mirror of
https://github.com/curl/curl.git
synced 2026-07-24 12:27:16 +03:00
runtests: add options to set minimum number of tests, use them
To detect mistakes made in the runtests framework that reduce the number of test runs. Before this patch it could go undetected with a green CI. The minimum thresholds will need light maintenance going forward (either bumping them periodically, or adjust if some may fell below minimums for justified reasons). We may also make minimums tighter or looser, or more job-specific. Latest number of test runs for each job can be seen at Test Clutch: https://testclutch.curl.se/static/reports/feature-matrix.html Also: - GHA: set minimums. Assisted-by: Dan Fandrich Follow-up tof2a75a14ddFollow-up tobb1391f943#19510 Closes #19942
This commit is contained in:
parent
eb39fee40b
commit
3f1cd809ee
9 changed files with 66 additions and 22 deletions
|
|
@ -50,6 +50,7 @@ BEGIN {
|
|||
$LOCKDIR
|
||||
$LOGDIR
|
||||
$maxtime
|
||||
$mintotal
|
||||
$memanalyze
|
||||
$MEMDUMP
|
||||
$perlcmd
|
||||
|
|
@ -104,6 +105,7 @@ our $CURLVERSION=""; # curl's reported version number
|
|||
our $CURLVERNUM=""; # curl's reported version number (without -DEV)
|
||||
our $randseed = 0; # random number seed
|
||||
our $maxtime; # curl command timeout override
|
||||
our $mintotal; # minimum number of tests to run
|
||||
|
||||
# paths
|
||||
our $pwd = getcwd(); # current working directory
|
||||
|
|
|
|||
|
|
@ -2403,6 +2403,10 @@ while(@ARGV) {
|
|||
my ($num)=($1);
|
||||
$maxtime=$num;
|
||||
}
|
||||
elsif($ARGV[0] =~ /--min=(\d+)/) {
|
||||
my ($num)=($1);
|
||||
$mintotal=$num;
|
||||
}
|
||||
elsif($ARGV[0] eq "-n") {
|
||||
# no valgrind
|
||||
undef $valgrind;
|
||||
|
|
@ -2546,6 +2550,7 @@ Usage: runtests.pl [options] [test selection(s)]
|
|||
-L path require an additional perl library file to replace certain functions
|
||||
-l list all test case names/descriptions
|
||||
-m=[seconds] set timeout for curl commands in tests
|
||||
--min=[count] minimum number of tests to run.
|
||||
-n no valgrind
|
||||
--no-debuginfod disable the valgrind debuginfod functionality
|
||||
-o variable=value set internal variable to the specified value
|
||||
|
|
@ -3332,6 +3337,19 @@ else {
|
|||
}
|
||||
}
|
||||
|
||||
if(!$mintotal && $ENV{"CURL_TEST_MIN"}) {
|
||||
$mintotal = $ENV{"CURL_TEST_MIN"};
|
||||
}
|
||||
if($mintotal) {
|
||||
if($total < $mintotal) {
|
||||
logmsg "TESTFAIL: number of tests run was below the minimum of: $mintotal\n";
|
||||
exit 1;
|
||||
}
|
||||
else {
|
||||
logmsg "TESTDONE: minimum number of tests was met: $mintotal\n";
|
||||
}
|
||||
}
|
||||
|
||||
if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
|
||||
exit 1;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue