romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-04-14 21:31:42 +03:00
Code Issues Projects Releases Packages Wiki Activity

runtests: add options to set minimum number of tests, use them

Browse source 
To detect mistakes made in the runtests framework that reduce
the number of test runs. Before this patch it could go undetected with
a green CI.

The minimum thresholds will need light maintenance going forward (either
bumping them periodically, or adjust if some may fell below minimums for
justified reasons). We may also make minimums tighter or looser, or more
job-specific.

Latest number of test runs for each job can be seen at Test Clutch:
https://testclutch.curl.se/static/reports/feature-matrix.html

Also:
- GHA: set minimums.

Assisted-by: Dan Fandrich

Follow-up to f2a75a14dd
Follow-up to bb1391f943 #19510

Closes #19942
This commit is contained in:
Viktor Szakats 2025-12-12 01:55:30 +01:00
parent eb39fee40b
commit 3f1cd809ee
No known key found for this signature in database
GPG key ID: B5ABD165E2AEF201
9 changed files with 66 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/distcheck.yml vendored
View file

@ -20,6 +20,7 @@ concurrency:
permissions: {}
env:
CURL_TEST_MIN: 1450
MAKEFLAGS: -j 5
jobs:

1
.github/workflows/linux-old.yml vendored
View file

@ -52,6 +52,7 @@ permissions: {}
env:
MAKEFLAGS: -j 5
CURL_CI: github
CURL_TEST_MIN: 1500
DEBIAN_FRONTEND: noninteractive
jobs:

10
.github/workflows/linux.yml vendored
View file

@ -36,6 +36,7 @@ permissions: {}
env:
MAKEFLAGS: -j 5
CURL_CI: github
CURL_TEST_MIN: 1600
CURL_CLANG_TIDYFLAGS: '-checks=-clang-analyzer-security.insecureAPI.bzero,-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.ArrayBound,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,-clang-analyzer-valist.Uninitialized'
# renovate: datasource=github-tags depName=libressl/portable versioning=semver registryUrl=https://github.com
LIBRESSL_VERSION: 4.2.1
@ -184,6 +185,7 @@ jobs:
configure: --with-openssl --disable-ipv6 --enable-debug --disable-unity --disable-libcurl-option --disable-digest-auth
- name: 'openssl https-only'
tflags: '--min=1150'
configure: >-
--with-openssl --enable-debug --disable-unity
--disable-dict --disable-gopher --disable-ldap --disable-telnet
@ -195,13 +197,13 @@ jobs:
- name: 'openssl torture !FTP'
install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev
generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON
tflags: '-t --shallow=25 !FTP'
tflags: '-t --shallow=25 !FTP --min=1450'
torture: true
- name: 'openssl torture FTP'
install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev
generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON
tflags: '-t --shallow=20 FTP'
tflags: '-t --shallow=20 FTP --min=260'
torture: true
- name: 'openssl i686'
@ -215,6 +217,7 @@ jobs:
--with-openssl --with-librtmp --with-libssh2 --with-libidn2 --enable-ares --enable-debug
- name: '!ssl !http !smtp !imap'
tflags: '--min=475'
configure: --without-ssl --enable-debug --disable-http --disable-smtp --disable-imap --disable-unity
- name: 'libressl Fil-C'
@ -303,11 +306,12 @@ jobs:
LDFLAGS: -fsanitize=memory
LIBS: -ldl
configure: CC=clang --without-ssl --without-zlib --without-brotli --without-zstd --without-libpsl --without-nghttp2 --enable-debug
tflags: '--min=1480'
- name: 'event-based'
install_packages: libssh-dev
configure: --enable-debug --disable-shared --disable-threaded-resolver --with-libssh --with-openssl
tflags: '-n --test-event'
tflags: '-n --test-event --min=1350'
- name: 'duphandle'
install_packages: libssh-dev

14
.github/workflows/macos.yml vendored
View file

@ -44,6 +44,7 @@ permissions: {}
env:
CURL_CI: github
CURL_TEST_MIN: 1700
MAKEFLAGS: -j 4
LDFLAGS: -w # suppress 'object file was built for newer macOS version than being linked' warnings
@ -225,16 +226,20 @@ jobs:
compiler: clang
install: brotli zstd
configure: --without-ssl --with-brotli --with-zstd
tflags: '--min=1450'
xcode: '' # default Xcode. Set it once to silence actionlint.
- name: '!ssl !debug'
compiler: gcc-13
configure: --without-ssl
tflags: '--min=950'
- name: '!ssl'
compiler: clang
configure: --enable-debug --without-ssl
tflags: '--min=1500'
- name: '!ssl libssh2 AppleIDN'
compiler: clang
configure: --enable-debug --with-libssh2=/opt/homebrew/opt/libssh2 --without-ssl --with-apple-idn
tflags: '--min=1550'
- name: 'OpenSSL libssh c-ares'
compiler: clang
install: libssh
@ -246,8 +251,10 @@ jobs:
- name: '!ssl c-ares'
compiler: clang
configure: --enable-debug --enable-ares --without-ssl
tflags: '--min=1500'
- name: '!ssl HTTP-only'
compiler: clang
tflags: '--min=950'
configure: >-
--enable-debug
--disable-alt-svc --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap
@ -277,7 +284,7 @@ jobs:
- name: 'OpenSSL event-based'
compiler: clang
configure: --enable-debug --with-openssl=/opt/homebrew/opt/openssl
tflags: --test-event
tflags: '--test-event --min=1300'
# cmake
- name: 'OpenSSL gsasl rtmp AppleIDN SecTrust +examples'
install: libnghttp3 libngtcp2 gsasl rtmpdump
@ -328,18 +335,19 @@ jobs:
compiler: clang
install: rustls-ffi
generate: -DENABLE_DEBUG=ON -DCURL_USE_RUSTLS=ON -DUSE_ECH=ON -DCURL_DISABLE_LDAP=ON
tflags: '--min=1650'
- name: 'OpenSSL torture !FTP'
compiler: clang
install: libnghttp3
install_steps: torture
generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON
tflags: -t --shallow=25 !FTP
tflags: '-t --shallow=25 !FTP --min=1450'
- name: 'OpenSSL torture FTP'
compiler: clang
install: libnghttp3
install_steps: torture
generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON
tflags: -t --shallow=20 FTP
tflags: '-t --shallow=20 FTP --min=260'
exclude:
# opt out jobs from combinations that have the compiler set manually
- { compiler: llvm@18, build: { compiler: 'clang' } }

1
.github/workflows/non-native.yml vendored
View file

@ -37,6 +37,7 @@ permissions: {}
env:
CURL_CI: github
CURL_TEST_MIN: 1750
jobs:
netbsd:

34
.github/workflows/windows.yml vendored
View file

@ -37,6 +37,7 @@ permissions: {}
env:
CURL_CI: github
CURL_TEST_MIN: 1700
jobs:
cygwin:
@ -197,26 +198,27 @@ jobs:
matrix:
include:
# MSYS
- { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' }
- { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' }
- { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' }
- { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' }
- { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '--min=1550', config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' }
- { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' }
- { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' }
- { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' }
# MinGW
- { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' }
- { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' }
- { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' }
- { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' }
- { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' }
- { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' }
- { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '--min=1650', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' }
# MinGW torture
- { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP --min=1300', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' }
# WARNING: libssh uses hard-coded world-writable paths (/etc/..., ~/.ssh/) to
# read its configuration from, making it vulnerable to attacks on
# Windows. Do not use this component till there is a fix for these.
# https://github.com/curl/curl-for-win/blob/3951808deb04df9489ee17430f236ed54436f81a/libssh.sh#L6-L8
- { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' }
- { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' }
- { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' }
- { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
# { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
- { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' }
- { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' }
- { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' }
- { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' }
- { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' }
- { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
# { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
- { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' }
- { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' }
fail-fast: false
steps:
- uses: msys2/setup-msys2@fb197b72ce45fb24f17bf3f807a388985654d1f2 # v2.29.0
@ -415,6 +417,7 @@ jobs:
run:
shell: msys2 {0}
env:
CURL_TEST_MIN: 1550
MAKEFLAGS: -j 5
MATRIX_DIR: '${{ matrix.dir }}'
strategy:
@ -775,6 +778,7 @@ jobs:
type: 'Debug'
image: 'windows-11-arm'
openssh: 'OpenSSH-Windows'
tflags: '--min=1650'
config: >-
-DENABLE_DEBUG=ON
-DCURL_USE_SCHANNEL=ON

7
docs/runtests.md
View file

@ -182,7 +182,12 @@ Lists all test case names.
## `-m=[seconds]`
Set timeout for curl commands in tests
Set timeout for curl commands in tests.
## `--min=[count]`
Set the minimum number of tests to run. This overrides the `CURL_TEST_MIN`
environment variable, if any.
## `-n`

2
tests/globalconfig.pm
View file

@ -50,6 +50,7 @@ BEGIN {
$LOCKDIR
$LOGDIR
$maxtime
$mintotal
$memanalyze
$MEMDUMP
$perlcmd
@ -104,6 +105,7 @@ our $CURLVERSION=""; # curl's reported version number
our $CURLVERNUM=""; # curl's reported version number (without -DEV)
our $randseed = 0; # random number seed
our $maxtime; # curl command timeout override
our $mintotal; # minimum number of tests to run
# paths
our $pwd = getcwd(); # current working directory

18
tests/runtests.pl
View file

@ -2403,6 +2403,10 @@ while(@ARGV) {
my ($num)=($1);
$maxtime=$num;
}
elsif($ARGV[0] =~ /--min=(\d+)/) {
my ($num)=($1);
$mintotal=$num;
}
elsif($ARGV[0] eq "-n") {
# no valgrind
undef $valgrind;
@ -2546,6 +2550,7 @@ Usage: runtests.pl [options] [test selection(s)]
-L path require an additional perl library file to replace certain functions
-l list all test case names/descriptions
-m=[seconds] set timeout for curl commands in tests
--min=[count] minimum number of tests to run.
-n no valgrind
--no-debuginfod disable the valgrind debuginfod functionality
-o variable=value set internal variable to the specified value
@ -3332,6 +3337,19 @@ else {
}
}
if(!$mintotal && $ENV{"CURL_TEST_MIN"}) {
$mintotal = $ENV{"CURL_TEST_MIN"};
}
if($mintotal) {
if($total < $mintotal) {
logmsg "TESTFAIL: number of tests run was below the minimum of: $mintotal\n";
exit 1;
}
else {
logmsg "TESTDONE: minimum number of tests was met: $mintotal\n";
}
}
if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
exit 1;
}