/ Check-in [4a5f6f1f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:The "make fuzztest" target now uses fuzzcheck instead of fuzzershell.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | test-using-fuzzcheck
Files: files | file ages | folders
SHA1: 4a5f6f1f0128657fd8d4d99d0682edd5bac2a19e
User & Date: drh 2015-05-26 18:15:08
Context
2015-05-26
18:58
Fix a one-byte buffer overread that may follow a syntax error while preparing an SQL statement. check-in: 07500393 user: dan tags: test-using-fuzzcheck
18:58
Fix fuzzcheck so that it responds correctly to the TEST_FAILURE environment variable. check-in: 76770c9e user: drh tags: test-using-fuzzcheck
18:15
The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. check-in: 4a5f6f1f user: drh tags: test-using-fuzzcheck
17:57
Enhance fuzzcheck with the ability to store descriptions in each source database and to run multiple source databases in a single invocation. check-in: 193364c8 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to Makefile.in.

518
519
520
521
522
523
524






525
526
527
528
529
530
531
...
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
#
TESTPROGS = \
  testfixture$(TEXE) \
  sqlite3$(TEXE) \
  sqlite3_analyzer$(TEXE) \
  sqldiff$(TEXE)








# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)

Makefile: $(TOP)/Makefile.in
................................................................................
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	$(TESTPROGS) fuzzoomtest
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not everything.
fulltestonly:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzershell$(TEXE)
	./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt

fuzzoomtest:	fuzzershell$(TEXE)
	./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom

# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzershell$(TEXE)
	valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt
	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzershell$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/main.test

sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
	echo "#define TCLSH 2" > $@
	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
	echo "static const char *tclsh_main_loop(void){" >> $@







>
>
>
>
>
>







 







|







|
|
<
<
<










|
|






|







518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
...
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993



994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
#
TESTPROGS = \
  testfixture$(TEXE) \
  sqlite3$(TEXE) \
  sqlite3_analyzer$(TEXE) \
  sqldiff$(TEXE)

# Databases containing fuzzer test cases
#
FUZZDATA = \
  $(TOP)/test/fuzzdata1.db \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)

Makefile: $(TOP)/Makefile.in
................................................................................
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not everything.
fulltestonly:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzcheck$(TEXE)
	./fuzzcheck$(TEXE) $(FUZZDATA)




# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzcheck$(TEXE)
	valgrind -v ./fuzzcheck$(TEXE) $(FUZZDATA)
	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/main.test

sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
	echo "#define TCLSH 2" > $@
	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
	echo "static const char *tclsh_main_loop(void){" >> $@

Changes to Makefile.msc.

1186
1187
1188
1189
1190
1191
1192







1193
1194
1195
1196
1197
1198
1199
....
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
#
TESTPROGS = \
  testfixture.exe \
  sqlite3.exe \
  sqlite3_analyzer.exe \
  sqldiff.exe









# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib

libsqlite3.lib:	$(LIBOBJ)
................................................................................

extensiontest: testfixture.exe testloadext.dll
	.\testfixture.exe $(TOP)\test\loadext.test

fulltest:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\all.test

soaktest:	$(TESTPROGS) fuzzoomtest
	.\testfixture.exe $(TOP)\test\all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner

fuzztest:	fuzzershell.exe
	.\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt

fuzzoomtest:	fuzzershell.exe
	.\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom

test:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\veryquick.test

smoketest:	$(TESTPROGS) fuzzershell.exe
	.\testfixture.exe $(TOP)\test\main.test

sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
	echo #define TCLSH 2 > $@
	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
	echo static const char *tclsh_main_loop(void){ >> $@







>
>
>
>
>
>
>







 







|








|
|
<
<
<




|







1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
....
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680



1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
#
TESTPROGS = \
  testfixture.exe \
  sqlite3.exe \
  sqlite3_analyzer.exe \
  sqldiff.exe

# Databases containing fuzzer test cases
#
FUZZDATA = \
  $(TOP)\test\fuzzdata1.db \
  $(TOP)\test\fuzzdata2.db \
  $(TOP)\test\fuzzdata3.db


# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib

libsqlite3.lib:	$(LIBOBJ)
................................................................................

extensiontest: testfixture.exe testloadext.dll
	.\testfixture.exe $(TOP)\test\loadext.test

fulltest:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\all.test

soaktest:	$(TESTPROGS)
	.\testfixture.exe $(TOP)\test\all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner

fuzztest:	fuzzcheck.exe
	.\fuzzcheck.exe $(FUZZDATA)




test:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\veryquick.test

smoketest:	$(TESTPROGS)
	.\testfixture.exe $(TOP)\test\main.test

sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
	echo #define TCLSH 2 > $@
	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
	echo static const char *tclsh_main_loop(void){ >> $@

Changes to main.mk.

398
399
400
401
402
403
404







405
406
407
408
409
410
411
...
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
# executables needed for testing
#
TESTPROGS = \
  testfixture$(EXE) \
  sqlite3$(EXE) \
  sqlite3_analyzer$(EXE) \
  sqldiff$(EXE)








# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.a sqlite3$(EXE)

libsqlite3.a:	$(LIBOBJ)
................................................................................
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	$(TESTPROGS) fuzzoomtest
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner

fuzztest:	fuzzershell$(EXE)
	./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt

fuzzoomtest:	fuzzershell$(EXE)
	./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom

test:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzershell$(EXE)
	valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt
	OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzershell$(EXE)
	./testfixture$(EXE) $(TOP)/test/main.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \







>
>
>
>
>
>
>







 







|








|
|
<
<
<







|
|






|







398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
...
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677



678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
# executables needed for testing
#
TESTPROGS = \
  testfixture$(EXE) \
  sqlite3$(EXE) \
  sqlite3_analyzer$(EXE) \
  sqldiff$(EXE)

# Databases containing fuzzer test cases
#
FUZZDATA = \
  $(TOP)/test/fuzzdata1.db \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.a sqlite3$(EXE)

libsqlite3.a:	$(LIBOBJ)
................................................................................
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	$(TESTPROGS)
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner

fuzztest:	fuzzcheck$(EXE) $(FUZZDATA)
	./fuzzcheck$(EXE) $(FUZZDATA)




test:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzcheck$(EXE) $(FUZZDATA)
	valgrind -v ./fuzzcheck$(EXE) $(FUZZDATA)
	OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzcheck$(EXE)
	./testfixture$(EXE) $(TOP)/test/main.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \

Added test/fuzzdata1.db.

cannot compute difference between binary files

Deleted test/fuzzdata1.txt.

cannot compute difference between binary files

Added test/fuzzdata2.db.

cannot compute difference between binary files

Deleted test/fuzzdata2.txt.

cannot compute difference between binary files

Added test/fuzzdata3.db.

cannot compute difference between binary files

Deleted test/mkfuzzdata1.tcl.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
#!/usr/bin/tclsh
#
# Run this script in order to rebuild the fuzzdata1.txt file containing
# fuzzer data for the fuzzershell utility that is create by afl-fuzz.
#
# This script gathers all of the test cases identified by afl-fuzz and
# runs afl-cmin and afl-tmin over them all to try to generate a mimimum
# set of tests that cover all observed behavior.
# 
# Options:
#
#    --afl-bin DIR1             DIR1 contains the AFL binaries
#    --fuzzershell PATH         Full pathname of instrumented fuzzershell
#    --afl-data DIR3            DIR3 is the "-o" directory from afl-fuzz
#    -o FILE                    Write results into FILE
#
set AFLBIN {}
set FUZZERSHELL {}
set AFLDATA {}
set OUTFILE {}

proc usage {} {
  puts stderr "Usage: $::argv0 --afl-bin DIR --fuzzershell PATH\
                  --afl-data DIR -o FILE"
  exit 1
}
proc cmdlineerr {msg} {
  puts stderr $msg
  usage
}

for {set i 0} {$i<[llength $argv]} {incr i} {
  set x [lindex $argv $i]
  if {[string index $x 0]!="-"} {cmdlineerr "illegal argument: $x"}
  set x [string trimleft $x -]
  incr i
  if {$i>=[llength $argv]} {cmdlineerr "no argument on --$x"}
  set a [lindex $argv $i]
  switch -- $x {
     afl-bin {set AFLBIN $a}
     afl-data {set AFLDATA $a}
     fuzzershell {set FUZZERSHELL $a}
     o {set OUTFILE $a}
     default {cmdlineerr "unknown option: --$x"}
  }
}
proc checkarg {varname option} {
  set val [set ::$varname]
  if {$val==""} {cmdlineerr "required option missing: --$option"}
}
checkarg AFLBIN afl-bin
checkarg AFLDATA afl-data
checkarg FUZZERSHELL fuzzershell
checkarg OUTFILE o
proc checkexec {x} {
  if {![file exec $x]} {cmdlineerr "cannot find $x"}
}
checkexec $AFLBIN/afl-cmin
checkexec $AFLBIN/afl-tmin
checkexec $FUZZERSHELL
proc checkdir {x} {
  if {![file isdir $x]} {cmdlineerr "no such directory: $x"}
}
checkdir $AFLDATA/queue

proc progress {msg} {
  puts "******** $msg"
  flush stdout
}
progress "mkdir tmp1 tmp2"
file mkdir tmp1 tmp2
progress "copying test cases from $AFLDATA into tmp1..."
set n 0
foreach file [glob -nocomplain $AFLDATA/queue/id:*] {
  incr n
  file copy $file tmp1/$n
}
foreach file [glob -nocomplain $AFLDATA/crash*/id:*] {
  incr n
  file copy $file tmp1/$n
}
progress "total $n files copied."
progress "running: $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL"
exec $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL >&@ stdout
progress "afl-cmin complete."
#
# Experiments show that running afl-tmin is too slow for this application.
# And it doesn't really make the test cases that much smaller.  So let's
# just skip it.
#
# foreach file [glob tmp2/*] {
#   progress "$AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] $FUZZERSHELL"
#   exec $AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] \
#       $FUZZERSHELL >&@ stdout
# }
progress "generating final output into $OUTFILE"
set out [open $OUTFILE wb]
puts $out "# Test data for use with fuzzershell.  Automatically
# generated using $argv0.  This file contains binary data
#"
set n 0
foreach file [glob tmp2/*] {
  incr n
  puts -nonewline $out "/****<$n>****/"
  set in [open $file rb]
  puts -nonewline $out [read $in]
  close $in
}
close $out
progress "done.  $n test cases written to $OUTFILE"
progress "clean-up..."
file delete -force tmp1
progress "culled test cases left in the tmp2 directory"
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<