Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -1289,10 +1289,16 @@ # Do extra testing but not everything. fulltestonly: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing +# +# WARNING: When the "fuzztest" target is run by the testrunner.tcl script, +# it does not actually run this code. Instead, it schedules equivalent +# commands. Therefore, if this target is updated, then code in +# testrunner_data.tcl (search for "trd_fuzztest_data") must also be updated. +# fuzztest: fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) ./fuzzcheck$(TEXE) $(FUZZDATA) ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) Index: ext/fts5/test/fts5optimize2.test ================================================================== --- ext/fts5/test/fts5optimize2.test +++ ext/fts5/test/fts5optimize2.test @@ -7,11 +7,11 @@ # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # -# TESTRUNNER: slow +# TESTRUNNER: superslow # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5optimize2 @@ -40,25 +40,6 @@ do_execsql_test 1.2 { SELECT count(*) FROM t1('mno') } $nLoop -do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t2 USING fts5(x); - INSERT INTO t2(t2, rank) VALUES('pgsz', 32); -} - -do_test 2.1 { - for {set ii 0} {$ii < $nLoop} {incr ii} { - execsql { - INSERT INTO t2 VALUES('abc def ghi'); - INSERT INTO t2 VALUES('jkl mno pqr'); - INSERT INTO t2(t2, rank) VALUES('merge', -1); - } - } -} {} - -do_execsql_test 2.2 { - SELECT count(*) FROM t2('mno') -} $nLoop - finish_test ADDED ext/fts5/test/fts5optimize3.test Index: ext/fts5/test/fts5optimize3.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5optimize3.test @@ -0,0 +1,45 @@ +# 2023 Aug 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: superslow +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5optimize2 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set nLoop 2500 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2(t2, rank) VALUES('pgsz', 32); +} + +do_test 1.1 { + for {set ii 0} {$ii < $nLoop} {incr ii} { + execsql { + INSERT INTO t2 VALUES('abc def ghi'); + INSERT INTO t2 VALUES('jkl mno pqr'); + INSERT INTO t2(t2, rank) VALUES('merge', -1); + } + } +} {} + +do_execsql_test 1.2 { + SELECT count(*) FROM t2('mno') +} $nLoop + +finish_test Index: test/testrunner.tcl ================================================================== --- test/testrunner.tcl +++ test/testrunner.tcl @@ -57,11 +57,10 @@ $a0 njob ?NJOB? $a0 status where SWITCHES are: --jobs NUMBER-OF-JOBS - --fuzztest --zipvfs ZIPVFS-SOURCE-DIR Interesting values for PERMUTATION are: veryquick - a fast subset of the tcl test scripts. This is the default. @@ -81,15 +80,10 @@ If a PERMUTATION is specified and is followed by the path to a Tcl script instead of a list of patterns, then that single Tcl test script is run with the specified permutation. -The --fuzztest option is ignored if the PERMUTATION is "release". Otherwise, -if it is present, then "make -C fuzztest" is run as part of the tests, -where is the directory containing the testfixture binary used to -run the script. - The "status" and "njob" commands are designed to be run from the same directory as a running testrunner.tcl script that is running tests. The "status" command prints a report describing the current state and progress of the tests. The "njob" command may be used to query or modify the number of sub-processes the test script uses to run tests. @@ -155,23 +149,32 @@ set TRG(fuzztest) 0 ;# is the fuzztest option present. set TRG(zipvfs) "" ;# -zipvfs option, if any switch -nocase -glob -- $tcl_platform(os) { *darwin* { - set TRG(platform) osx - set TRG(make) make.sh - set TRG(makecmd) "bash make.sh" + set TRG(platform) osx + set TRG(make) make.sh + set TRG(makecmd) "bash make.sh" + set TRG(testfixture) testfixture + set TRG(run) run.sh + set TRG(runcmd) "bash run.sh" } *linux* { - set TRG(platform) linux - set TRG(make) make.sh - set TRG(makecmd) "bash make.sh" + set TRG(platform) linux + set TRG(make) make.sh + set TRG(makecmd) "bash make.sh" + set TRG(testfixture) testfixture + set TRG(run) run.sh + set TRG(runcmd) "bash run.sh" } *win* { - set TRG(platform) win - set TRG(make) make.bat - set TRG(makecmd) make.bat + set TRG(platform) win + set TRG(make) make.bat + set TRG(makecmd) make.bat + set TRG(testfixture) testfixture.exe + set TRG(run) run.bat + set TRG(runcmd) "run.bat" } default { error "cannot determine platform!" } } @@ -179,33 +182,72 @@ #------------------------------------------------------------------------- # The database schema used by the testrunner.db database. # set TRG(schema) { - DROP TABLE IF EXISTS script; + DROP TABLE IF EXISTS jobs; DROP TABLE IF EXISTS config; - CREATE TABLE script( - build TEXT DEFAULT '', - config TEXT, - filename TEXT, -- full path to test script - slow BOOLEAN, -- true if script is "slow" + /* + ** This table contains one row for each job that testrunner.tcl must run + ** before the entire test run is finished. + ** + ** jobid: + ** Unique identifier for each job. Must be a +ve non-zero number. + ** + ** displaytype: + ** 3 or 4 letter mnemonic for the class of tests this belongs to e.g. + ** "fuzz", "tcl", "make" etc. + ** + ** displayname: + ** Name/description of job. For display purposes. + ** + ** build: + ** If the job requires a make.bat/make.sh make wrapper (i.e. to build + ** something), the name of the build configuration it uses. See + ** testrunner_data.tcl for a list of build configs. e.g. "Win32-MemDebug". + ** + ** dirname: + ** If the job should use a well-known directory name for its + ** sub-directory instead of an anonymous "testdir[1234...]" sub-dir + ** that is deleted after the job is finished. + ** + ** cmd: + ** Bash or batch script to run the job. + ** + ** depid: + ** The jobid value of a job that this job depends on. This job may not + ** be run before its depid job has finished successfully. + ** + ** priority: + ** Higher values run first. Sometimes. + */ + CREATE TABLE jobs( + /* Fields populated when db is initialized */ + jobid INTEGER PRIMARY KEY, -- id to identify job + displaytype TEXT NOT NULL, -- Type of test (for one line report) + displayname TEXT NOT NULL, -- Human readable job name + build TEXT NOT NULL DEFAULT '', -- make.sh/make.bat file request, if any + dirname TEXT NOT NULL DEFAULT '', -- directory name, if required + cmd TEXT NOT NULL, -- shell command to run + depid INTEGER, -- identifier of dependency (or '') + priority INTEGER NOT NULL, -- higher priority jobs may run earlier + + /* Fields updated as jobs run */ + starttime INTEGER, + endtime INTEGER, state TEXT CHECK( state IN ('', 'ready', 'running', 'done', 'failed') ), - time INTEGER, -- Time in ms - output TEXT, -- full output of test script - priority INTEGER, - jobtype TEXT CHECK( jobtype IN ('script', 'build', 'make') ), - PRIMARY KEY(build, config, filename) + output TEXT ); CREATE TABLE config( name TEXT COLLATE nocase PRIMARY KEY, value ) WITHOUT ROWID; - CREATE INDEX i1 ON script(state, jobtype); - CREATE INDEX i2 ON script(state, priority); + CREATE INDEX i1 ON jobs(state, priority); + CREATE INDEX i2 ON jobs(depid); } #------------------------------------------------------------------------- #-------------------------------------------------------------------------- # Check if this script is being invoked to run a single file. If so, @@ -301,43 +343,18 @@ # if {[llength $argv]==1 && [string compare -nocase status [lindex $argv 0]]==0 } { - proc display_job {build config filename {tm ""}} { - if {$config=="build"} { - set fname "build: $filename" - set config "" - } elseif {$config=="make"} { - set fname "make: $filename" - set config "" - } else { - set fname [file normalize $filename] - if {[string first $::srcdir $fname]==0} { - set fname [string range $fname [string length $::srcdir]+1 end] - } - } - set dfname [format %-33s $fname] - - set dbuild "" - set dconfig "" - set dparams "" + proc display_job {jobdict {tm ""}} { + array set job $jobdict + + set dfname [format %-60s $job(displayname)] + set dtm "" - if {$build!=""} { set dbuild $build } - if {$config!="" && $config!="full"} { set dconfig $config } - if {$dbuild!="" || $dconfig!=""} { - append dparams "(" - if {$dbuild!=""} {append dparams "build=$dbuild"} - if {$dbuild!="" && $dconfig!=""} {append dparams " "} - if {$dconfig!=""} {append dparams "config=$dconfig"} - append dparams ")" - set dparams [format %-33s $dparams] - } - if {$tm!=""} { - set dtm "\[${tm}ms\]" - } - puts " $dfname $dparams $dtm" + if {$tm!=""} { set dtm "\[[expr {$tm-$job(starttime)}]ms\]" } + puts " $dfname $dtm" } sqlite3 mydb $TRG(dbname) mydb timeout 1000 mydb eval BEGIN @@ -353,11 +370,11 @@ }] set total 0 foreach s {"" ready running done failed} { set S($s) 0 } mydb eval { - SELECT state, count(*) AS cnt FROM script GROUP BY 1 + SELECT state, count(*) AS cnt FROM jobs GROUP BY 1 } { incr S($state) $cnt incr total $cnt } set fin [expr $S(done)+$S(failed)] @@ -373,23 +390,21 @@ set srcdir [file dirname [file dirname $TRG(info_script)]] if {$S(running)>0} { puts "Running: " mydb eval { - SELECT build, config, filename, time FROM script WHERE state='running' - ORDER BY time - } { - display_job $build $config $filename [expr $now-$time] + SELECT * FROM jobs WHERE state='running' ORDER BY starttime + } job { + display_job [array get job] $now } } if {$S(failed)>0} { puts "Failures: " mydb eval { - SELECT build, config, filename FROM script WHERE state='failed' - ORDER BY 3 - } { - display_job $build $config $filename + SELECT * FROM jobs WHERE state='failed' ORDER BY starttime + } job { + display_job [array get job] } } mydb close exit @@ -406,12 +421,10 @@ if {[string range $a 0 0]=="-"} { if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} { incr ii set TRG(nJob) [lindex $argv $ii] if {$isLast} { usage } - } elseif {($n>2 && [string match "$a*" --fuzztest]) || $a=="-f"} { - set TRG(fuzztest) 1 } elseif {($n>2 && [string match "$a*" --zipvfs]) || $a=="-z"} { incr ii set TRG(zipvfs) [lindex $argv $ii] if {$isLast} { usage } } else { @@ -478,124 +491,16 @@ foreach f [glob -nocomplain [file join $dir *]] { catch { file delete -force $f } } } -proc copy_dir {from to} { - foreach f [glob -nocomplain [file join $from *]] { - catch { file copy -force $f $to } - } -} - proc build_to_dirname {bname} { set fold [string tolower [string map {- _} $bname]] return "testrunner_build_$fold" } #------------------------------------------------------------------------- -# Return a list of tests to run. Each element of the list is itself a -# list of two elements - the name of a permuations.test configuration -# followed by the full path to a test script. i.e.: -# -# {BUILD CONFIG FILENAME} {BUILD CONFIG FILENAME} ... -# -proc testset_patternlist {patternlist} { - global TRG - - set testset [list] ;# return value - - set first [lindex $patternlist 0] - - if {$first=="sdevtest" || $first=="mdevtest"} { - set CONFIGS(sdevtest) {All-Debug All-Sanitize} - set CONFIGS(mdevtest) {All-Debug All-O0} - - set patternlist [lrange $patternlist 1 end] - - foreach b $CONFIGS($first) { - lappend testset [list $b build testfixture] - lappend testset [list $b make fuzztest] - testset_append testset $b veryquick $patternlist - } - } elseif {$first=="release"} { - set platform $::TRG(platform) - - set patternlist [lrange $patternlist 1 end] - foreach b [trd_builds $platform] { - foreach c [trd_configs $platform $b] { - testset_append testset $b $c $patternlist - } - - if {[llength $patternlist]==0 || $b=="User-Auth"} { - set target testfixture - } else { - set target coretestprogs - } - lappend testset [list $b build $target] - } - - if {[llength $patternlist]==0} { - foreach b [trd_builds $platform] { - foreach e [trd_extras $platform $b] { - lappend testset [list $b make $e] - } - } - } - - set TRG(fuzztest) 0 ;# ignore --fuzztest option in this case - - } elseif {$first=="all"} { - - set clist [trd_all_configs] - set patternlist [lrange $patternlist 1 end] - foreach c $clist { - testset_append testset "" $c $patternlist - } - - } elseif {[info exists ::testspec($first)]} { - set clist $first - testset_append testset "" $first [lrange $patternlist 1 end] - } elseif { [llength $patternlist]==0 } { - testset_append testset "" veryquick $patternlist - } else { - testset_append testset "" full $patternlist - } - if {$TRG(fuzztest)} { - if {$TRG(platform)=="win"} { error "todo" } - lappend testset [list "" make fuzztest] - } - - set testset -} - -proc testset_append {listvar build config patternlist} { - upvar $listvar lvar - - catch { array unset O } - array set O $::testspec($config) - - foreach f $O(-files) { - if {[llength $patternlist]>0} { - set bMatch 0 - foreach p $patternlist { - if {[string match $p [file tail $f]]} { - set bMatch 1 - break - } - } - if {$bMatch==0} continue - } - - if {[file pathtype $f]!="absolute"} { - set f [file join $::testdir $f] - } - lappend lvar [list $build $config $f] - } -} - -#-------------------------------------------------------------------------- - proc r_write_db {tcl} { trdb eval { BEGIN EXCLUSIVE } uplevel $tcl trdb eval { COMMIT } @@ -613,148 +518,365 @@ set orderby "ORDER BY priority ASC" } else { set orderby "ORDER BY priority DESC" } + set ret [list] + r_write_db { - set f "" - set c "" - trdb eval " - SELECT build, config, filename - FROM script - WHERE state='ready' - $orderby LIMIT 1 - " { - set b $build - set c $config - set f $filename - } - if {$f!=""} { + set query " + SELECT * FROM jobs AS j WHERE state='ready' $orderby LIMIT 1 + " + trdb eval $query job { set tm [clock_milliseconds] set T($iJob) $tm - trdb eval { - UPDATE script SET state='running', time=$tm - WHERE (build, config, filename) = ($b, $c, $f) + set jobid $job(jobid) + + trdb eval { + UPDATE jobs SET starttime=$tm, state='running' WHERE jobid=$jobid } + + set ret [array get job] } } - if {$f==""} { return "" } - list $b $c $f + return $ret } #rename r_get_next_job r_get_next_job_r #proc r_get_next_job {iJob} { -# puts [time { set res [r_get_next_job_r $iJob] }] -# set res + #puts [time { set res [r_get_next_job_r $iJob] }] + #set res #} + +# Usage: +# +# add_job OPTION ARG OPTION ARG... +# +# where available OPTIONS are: +# +# -displaytype +# -displayname +# -build +# -dirname +# -cmd +# -depid +# -priority +# +# Returns the jobid value for the new job. +# +proc add_job {args} { + + set options { + -displaytype -displayname -build -dirname + -cmd -depid -priority + } + + # Set default values of options. + set A(-dirname) "" + set A(-depid) "" + set A(-priority) 0 + set A(-build) "" + + array set A $args + + # Check all required options are present. And that no extras are present. + foreach o $options { + if {[info exists A($o)]==0} { error "missing required option $o" } + } + foreach o [array names A] { + if {[lsearch -exact $options $o]<0} { error "unrecognized option: $o" } + } + + set state "" + if {$A(-depid)==""} { set state ready } + + trdb eval { + INSERT INTO jobs( + displaytype, displayname, build, dirname, cmd, depid, priority, + state + ) VALUES ( + $A(-displaytype), + $A(-displayname), + $A(-build), + $A(-dirname), + $A(-cmd), + $A(-depid), + $A(-priority), + $state + ) + } + + trdb last_insert_rowid +} + +proc add_tcl_jobs {build config patternlist} { + global TRG + + set topdir [file dirname $::testdir] + set testrunner_tcl [file normalize [info script]] + + if {$build==""} { + set testfixture [info nameofexec] + } else { + set testfixture [file join [lindex $build 1] $TRG(testfixture)] + } + if {[lindex $build 2]=="Valgrind"} { + set setvar "export OMIT_MISUSE=1\n" + set testfixture "${setvar}valgrind -v --error-exitcode=1 $testfixture" + } + + # The ::testspec array is populated by permutations.test + foreach f [dict get $::testspec($config) -files] { + + if {[llength $patternlist]>0} { + set bMatch 0 + foreach p $patternlist { + if {[string match $p [file tail $f]]} { + set bMatch 1 + break + } + } + if {$bMatch==0} continue + } + + if {[file pathtype $f]!="absolute"} { set f [file join $::testdir $f] } + set f [file normalize $f] + + set displayname [string map [list $topdir/ {}] $f] + if {$config=="full" || $config=="veryquick"} { + set cmd "$testfixture $f" + } else { + set cmd "$testfixture $testrunner_tcl $config $f" + set displayname "config=$config $displayname" + } + if {$build!=""} { + set displayname "[lindex $build 2] $displayname" + } + + set lProp [trd_test_script_properties $f] + set priority 0 + if {[lsearch $lProp slow]>=0} { set priority 2 } + if {[lsearch $lProp superslow]>=0} { set priority 4 } + + add_job \ + -displaytype tcl \ + -displayname $displayname \ + -cmd $cmd \ + -depid [lindex $build 0] \ + -priority $priority + + } +} + +proc add_build_job {buildname target} { + global TRG + + set dirname "[string tolower [string map {- _} $buildname]]_$target" + set dirname "testrunner_bld_$dirname" + + set id [add_job \ + -displaytype bld \ + -displayname "Build $buildname ($target)" \ + -dirname $dirname \ + -build $buildname \ + -cmd "$TRG(makecmd) $target" \ + -priority 3 + ] + + list $id [file normalize $dirname] $buildname +} + +proc add_make_job {bld target} { + global TRG + + if {$TRG(platform)=="win"} { + set path [string map {/ \\} [lindex $bld 1]] + set cmd "xcopy /S $path\\* ." + } else { + set cmd "cp -r [lindex $bld 1]/* ." + } + append cmd "\n$TRG(makecmd) $target" + + add_job \ + -displaytype make \ + -displayname "[lindex $bld 2] make $target" \ + -cmd $cmd \ + -depid [lindex $bld 0] \ + -priority 1 +} + +proc add_fuzztest_jobs {buildname} { + + foreach {interpreter scripts} [trd_fuzztest_data] { + set subcmd [lrange $interpreter 1 end] + set interpreter [lindex $interpreter 0] + + set bld [add_build_job $buildname $interpreter] + foreach {depid dirname displayname} $bld {} + + foreach s $scripts { + + # Fuzz data files fuzzdata1.db and fuzzdata2.db are larger than + # the others. So ensure that these are run as a higher priority. + set tail [file tail $s] + if {$tail=="fuzzdata1.db" || $tail=="fuzzdata2.db"} { + set priority 5 + } else { + set priority 1 + } + + add_job \ + -displaytype fuzz \ + -displayname "$buildname $interpreter $tail" \ + -depid $depid \ + -cmd "[file join $dirname $interpreter] $subcmd $s" \ + -priority $priority + } + } +} + +proc add_zipvfs_jobs {} { + global TRG + source [file join $TRG(zipvfs) test zipvfs_testrunner.tcl] + + set bld [add_build_job Zipvfs $TRG(testfixture)] + foreach s [zipvfs_testrunner_files] { + set cmd "[file join [lindex $bld 1] $TRG(testfixture)] $s" + add_job \ + -displaytype tcl \ + -displayname "Zipvfs [file tail $s]" \ + -cmd $cmd \ + -depid [lindex $bld 0] + } + + set ::env(SQLITE_TEST_DIR) $::testdir +} + +proc add_jobs_from_cmdline {patternlist} { + global TRG + + if {$TRG(zipvfs)!=""} { + add_zipvfs_jobs + if {[llength $patternlist]==0} return + } + + if {[llength $patternlist]==0} { + set patternlist [list veryquick] + } + + set first [lindex $patternlist 0] + switch -- $first { + all { + set patternlist [lrange $patternlist 1 end] + set clist [trd_all_configs] + foreach c $clist { + add_tcl_jobs "" $c $patternlist + } + } + + mdevtest { + foreach b [list All-O0 All-Debug] { + set bld [add_build_job $b $TRG(testfixture)] + add_tcl_jobs $bld veryquick "" + add_fuzztest_jobs $b + } + } + + sdevtest { + foreach b [list All-Sanitize All-Debug] { + set bld [add_build_job $b $TRG(testfixture)] + add_tcl_jobs $bld veryquick "" + add_fuzztest_jobs $b + } + } + + release { + foreach b [trd_builds $TRG(platform)] { + set bld [add_build_job $b $TRG(testfixture)] + foreach c [trd_configs $TRG(platform) $b] { + add_tcl_jobs $bld $c "" + } + + foreach e [trd_extras $TRG(platform) $b] { + if {$e=="fuzztest"} { + add_fuzztest_jobs $b + } else { + add_make_job $bld $e + } + } + } + } + + default { + if {[info exists ::testspec($first)]} { + add_tcl_jobs "" $first [lrange $patternlist 1 end] + } else { + add_tcl_jobs "" full $patternlist + } + } + } +} proc make_new_testset {} { global TRG - set tests [list] - if {$TRG(zipvfs)!=""} { - source [file join $TRG(zipvfs) test zipvfs_testrunner.tcl] - lappend tests {*}[zipvfs_testrunner_testset] - } - - if {$tests=="" || $TRG(patternlist)!=""} { - lappend tests {*}[testset_patternlist $TRG(patternlist)] - } - r_write_db { - trdb eval $TRG(schema) set nJob $TRG(nJob) set cmdline $TRG(cmdline) set tm [clock_milliseconds] trdb eval { REPLACE INTO config VALUES('njob', $nJob ); } trdb eval { REPLACE INTO config VALUES('cmdline', $cmdline ); } trdb eval { REPLACE INTO config VALUES('start', $tm ); } - foreach t $tests { - foreach {b c s} $t {} - set slow 0 - - if {$c!="make" && $c!="build"} { - set fd [open $s] - for {set ii 0} {$ii<100 && ![eof $fd]} {incr ii} { - set line [gets $fd] - if {[string match -nocase *testrunner:* $line]} { - regexp -nocase {.*testrunner:(.*)} $line -> properties - foreach p $properties { - if {$p=="slow"} { set slow 1 } - if {$p=="superslow"} { set slow 2 } - } - } - } - close $fd - } - - if {$c=="make" && $b==""} { - # --fuzztest option - set slow 1 - } - - if {$c=="veryquick"} { - set c "" - } - - set state ready - if {$b!="" && $c!="build"} { - set state "" - } - - set priority [expr {$slow*2}] - if {$c=="make"} { incr priority 3 } - if {$c=="build"} { incr priority 1 } - - if {$c=="make" || $c=="build"} { - set jobtype $c - } else { - set jobtype "script" - } - - trdb eval { - INSERT INTO script - (build, config, filename, slow, state, priority, jobtype) - VALUES ($b, $c, $s, $slow, $state, $priority, $jobtype) - } - } - } -} - -proc script_input_ready {fd iJob b c f} { + add_jobs_from_cmdline $TRG(patternlist) + } + +} + +proc script_input_ready {fd iJob jobid} { global TRG global O global T if {[eof $fd]} { + trdb eval { SELECT * FROM jobs WHERE jobid=$jobid } job {} + + # If this job specified a directory name, then delete the run.sh/run.bat + # file from it before continuing. This is because the contents of this + # directory might be copied by some other job, and we don't want to copy + # the run.sh file in this case. + if {$job(dirname)!=""} { + file delete -force [file join $job(dirname) $TRG(run)] + } + set ::done 1 fconfigure $fd -blocking 1 set state "done" set rc [catch { close $fd } msg] if {$rc} { - puts "FAILED: $b $c $f" + if {[info exists TRG(reportlength)]} { + puts -nonewline "[string repeat " " $TRG(reportlength)]\r" + } + puts "FAILED: $job(displayname) ($iJob)" set state "failed" } - set tm [expr [clock_milliseconds] - $T($iJob)] + set tm [clock_milliseconds] + set jobtm [expr {$tm - $job(starttime)}] - puts $TRG(log) "### $b ### $c ### $f ${tm}ms ($state)" + puts $TRG(log) "### $job(displayname) ${jobtm}ms ($state)" puts $TRG(log) [string trim $O($iJob)] r_write_db { set output $O($iJob) trdb eval { - UPDATE script SET output = $output, state=$state, time=$tm - WHERE (build, config, filename) = ($b, $c, $f) - } - if {$state=="done" && $c=="build"} { - trdb eval { - UPDATE script SET state = 'ready' WHERE (build, state)==($b, '') - } + UPDATE jobs + SET output=$output, state=$state, endtime=$tm + WHERE jobid=$jobid; + UPDATE jobs SET state='ready' WHERE depid=$jobid; } } dirs_freeDir $iJob launch_some_jobs @@ -781,83 +903,43 @@ global T set testfixture [info nameofexec] set script $TRG(info_script) - set dir [dirname $iJob] - create_or_clear_dir $dir - set O($iJob) "" - set job [r_get_next_job $iJob] - if {$job==""} { return 0 } - - foreach {b c f} $job {} - - if {$c=="build"} { - set testdir [file dirname $TRG(info_script)] - set srcdir [file dirname $testdir] - set builddir [build_to_dirname $b] - create_or_clear_dir $builddir - - if {$b=="Zipvfs"} { + set jobdict [r_get_next_job $iJob] + if {$jobdict==""} { return 0 } + array set job $jobdict + + set dir $job(dirname) + if {$dir==""} { set dir [dirname $iJob] } + create_or_clear_dir $dir + + if {$job(build)!=""} { + set srcdir [file dirname $::testdir] + if {$job(build)=="Zipvfs"} { set script [zipvfs_testrunner_script] } else { - set script [trd_buildscript $b $srcdir [expr {$TRG(platform)=="win"}]] + set bWin [expr {$TRG(platform)=="win"}] + set script [trd_buildscript $job(build) $srcdir $bWin] } - - set fd [open [file join $builddir $TRG(make)] w] + set fd [open [file join $dir $TRG(make)] w] puts $fd $script close $fd - - puts "Launching build \"$b\" in directory $builddir..." - set target coretestprogs - if {$b=="User-Auth"} { set target testfixture } - - set cmd "$TRG(makecmd) $target" - set dir $builddir - - } elseif {$c=="make"} { - if {$b==""} { - if {$f!="fuzztest"} { error "corruption in testrunner.db!" } - # Special case - run [make fuzztest] - set makedir [file dirname $testfixture] - if {$TRG(platform)=="win"} { - error "how?" - } else { - set cmd [list make -C $makedir fuzztest] - } - } else { - set builddir [build_to_dirname $b] - copy_dir $builddir $dir - set cmd "$TRG(makecmd) $f" - } - } else { - if {$b==""} { - set testfixture [info nameofexec] - } else { - set tail testfixture - if {$TRG(platform)=="win"} { set tail testfixture.exe } - set testfixture [file normalize [file join [build_to_dirname $b] $tail]] - } - - if {$c=="valgrind"} { - set testfixture "valgrind -v --error-exitcode=1 $testfixture" - set ::env(OMIT_MISUSE) 1 - } - set cmd [concat $testfixture [list $script $c $f]] } set pwd [pwd] cd $dir - set fd [open "|$cmd 2>@1" r] + set fd [open $TRG(run) w] + puts $fd $job(cmd) + close $fd + set fd [open "|$TRG(runcmd) 2>@1" r] cd $pwd - set pid [pid $fd] fconfigure $fd -blocking false - fileevent $fd readable [list script_input_ready $fd $iJob $b $c $f] - unset -nocomplain ::env(OMIT_MISUSE) + fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)] return 1 } proc one_line_report {} { @@ -864,43 +946,34 @@ global TRG set tm [expr [clock_milliseconds] - $TRG(starttime)] set tm [format "%d" [expr int($tm/1000.0 + 0.5)]] - foreach s {ready running done failed} { - set v($s,build) 0 - set v($s,make) 0 - set v($s,script) 0 - } - r_write_db { - trdb eval { - SELECT state, jobtype, count(*) AS cnt - FROM script - GROUP BY state, jobtype + trdb eval { + SELECT displaytype, state, count(*) AS cnt + FROM jobs + GROUP BY 1, 2 } { - set v($state,$jobtype) $cnt - if {[info exists t($jobtype)]} { - incr t($jobtype) $cnt - } else { - set t($jobtype) $cnt - } + set v($state,$displaytype) $cnt + incr t($displaytype) $cnt } } set text "" - foreach j [array names t] { + foreach j [lsort [array names t]] { + foreach k {done failed running} { incr v($k,$j) 0 } set fin [expr $v(done,$j) + $v(failed,$j)] lappend text "$j ($fin/$t($j)) f=$v(failed,$j) r=$v(running,$j)" } if {[info exists TRG(reportlength)]} { puts -nonewline "[string repeat " " $TRG(reportlength)]\r" } set report "${tm}s: [join $text { }]" set TRG(reportlength) [string length $report] - if {[string length $report]<80} { + if {[string length $report]<100} { puts -nonewline "$report\r" flush stdout } else { puts $report } @@ -908,13 +981,12 @@ after $TRG(reporttime) one_line_report } proc launch_some_jobs {} { global TRG - r_write_db { - set nJob [trdb one { SELECT value FROM config WHERE name='njob' }] - } + set nJob [trdb one { SELECT value FROM config WHERE name='njob' }] + while {[dirs_nHelper]<$nJob} { set iDir [dirs_allocDir] if {0==[launch_another_job $iDir]} { dirs_freeDir $iDir break; @@ -928,11 +1000,10 @@ set TRG(starttime) [clock_milliseconds] set TRG(log) [open $TRG(logname) w] launch_some_jobs - # launch_another_job $ii one_line_report while {[dirs_nHelper]>0} { after 500 {incr ::wakeup} vwait ::wakeup @@ -941,17 +1012,17 @@ one_line_report r_write_db { set tm [clock_milliseconds] trdb eval { REPLACE INTO config VALUES('end', $tm ); } - set nErr [trdb one {SELECT count(*) FROM script WHERE state='failed'}] + set nErr [trdb one {SELECT count(*) FROM jobs WHERE state='failed'}] if {$nErr>0} { puts "$nErr failures:" trdb eval { - SELECT build, config, filename FROM script WHERE state='failed' + SELECT displayname FROM jobs WHERE state='failed' } { - puts "FAILED: $build $config $filename" + puts "FAILED: $displayname" } } } puts "\nTest database is $TRG(dbname)" Index: test/testrunner_data.tcl ================================================================== --- test/testrunner_data.tcl +++ test/testrunner_data.tcl @@ -368,18 +368,48 @@ set clist } proc trd_extras {platform bld} { trd_import + if {[info exists extra($platform.$bld)]==0} { return [list] } + return $extra($platform.$bld) +} + +# Usage: +# +# trd_fuzztest_data +# +# This returns data used by testrunner.tcl to run commands equivalent +# to [make fuzztest]. The returned value is a list, which should be +# interpreted as a sequence of pairs. The first element of each pair +# is an interpreter name. The second element is a list of files. +# testrunner.tcl automatically creates one job to build each interpreter, +# and one to run each of the files with it once it has been built. +# +# In practice, the returned value looks like this: +# +# { +# {fuzzcheck {$testdir/fuzzdata1.db $testdir/fuzzdata2.db ...}} +# {{sessionfuzz run} $testdir/sessionfuzz-data1.db} +# } +# +# where $testdir is replaced by the full-path to the test-directory (the +# directory containing this file). "fuzzcheck" and "sessionfuzz" have .exe +# extensions on windows. +# +proc trd_fuzztest_data {} { + set EXE "" + set lFuzzDb [glob [file join $::testdir fuzzdata*.db]] + set lSessionDb [glob [file join $::testdir sessionfuzz-data*.db]] - set elist [list] - if {[info exists extra($platform.$bld)]} { - set elist $extra($platform.$bld) + if {$::tcl_platform(platform)=="windows"} { + return [list fuzzcheck.exe $lFuzzDb] } - set elist + return [list fuzzcheck $lFuzzDb sessionfuzz $lSessionDb] } + proc trd_all_configs {} { trd_import set all_configs } @@ -392,11 +422,11 @@ set tcldir [::tcl::pkgconfig get libdir,install] set myopts "" if {[info exists ::env(OPTS)]} { append myopts "# From environment variable:\n" - append myopts "OPTS=$::env(OPTS)\n" + append myopts "OPTS=$::env(OPTS)\n\n" } foreach o [lsort $opts] { append myopts "OPTS=\"\$OPTS $o\"\n" } @@ -507,23 +537,30 @@ lappend opts -DSQLITE_DEBUG } default { error "Cannot translate $param for MSVC" } - } } continue } if {[string range $param 0 0]=="-"} { - if {$bMsvc && [regexp -- {^-O(\d+)$} $param -> level]} { - lappend makeOpts OPTIMIZATIONS=$level - } else { - lappend cflags $param + + if {$bMsvc} { + if {[regexp -- {^-O(\d+)$} $param -> level]} { + lappend makeOpts OPTIMIZATIONS=$level + continue + } + if {$param eq "-fsanitize=address,undefined"} { + lappend makeOpts ASAN=1 + continue + } } + + lappend cflags $param continue } lappend makeOpts $param } @@ -558,6 +595,38 @@ # Generate and return the script. return [make_script $build($config) $srcdir $bMsvc] } +# Usage: +# +# trd_test_script_properties PATH +# +# The argument must be a path to a Tcl test script. This function scans the +# first 100 lines of the script for lines that look like: +# +# TESTRUNNER: +# +# where is a list of identifiers, each of which defines a +# property of the test script. Example properties are "slow" or "superslow". +# +proc trd_test_script_properties {path} { + # Use this global array as a cache: + global trd_test_script_properties_cache + + if {![info exists trd_test_script_properties_cache($path)]} { + set fd [open $path] + set ret [list] + for {set line 0} {$line < 100 && ![eof $fd]} {incr line} { + set text [gets $fd] + if {[string match -nocase *testrunner:* $text]} { + regexp -nocase {.*testrunner:(.*)} $text -> properties + lappend ret {*}$properties + } + } + set trd_test_script_properties_cache($path) $ret + close $fd + } + + set trd_test_script_properties_cache($path) +}