diff --git a/.github/workflows/distcheck.yml b/.github/workflows/distcheck.yml index 86c05fd0f0..2533f30264 100644 --- a/.github/workflows/distcheck.yml +++ b/.github/workflows/distcheck.yml @@ -20,6 +20,7 @@ concurrency: permissions: {} env: + CURL_TEST_MIN: 1450 MAKEFLAGS: -j 5 jobs: diff --git a/.github/workflows/linux-old.yml b/.github/workflows/linux-old.yml index e282b1c471..d4a3071e64 100644 --- a/.github/workflows/linux-old.yml +++ b/.github/workflows/linux-old.yml @@ -52,6 +52,7 @@ permissions: {} env: MAKEFLAGS: -j 5 CURL_CI: github + CURL_TEST_MIN: 1500 DEBIAN_FRONTEND: noninteractive jobs: diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3a9c35fe38..47bb933099 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -36,6 +36,7 @@ permissions: {} env: MAKEFLAGS: -j 5 CURL_CI: github + CURL_TEST_MIN: 1600 CURL_CLANG_TIDYFLAGS: '-checks=-clang-analyzer-security.insecureAPI.bzero,-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.ArrayBound,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,-clang-analyzer-valist.Uninitialized' # renovate: datasource=github-tags depName=libressl/portable versioning=semver registryUrl=https://github.com LIBRESSL_VERSION: 4.2.1 @@ -184,6 +185,7 @@ jobs: configure: --with-openssl --disable-ipv6 --enable-debug --disable-unity --disable-libcurl-option --disable-digest-auth - name: 'openssl https-only' + tflags: '--min=1150' configure: >- --with-openssl --enable-debug --disable-unity --disable-dict --disable-gopher --disable-ldap --disable-telnet @@ -195,13 +197,13 @@ jobs: - name: 'openssl torture !FTP' install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON - tflags: '-t --shallow=25 !FTP' + tflags: '-t --shallow=25 !FTP --min=1450' torture: true - name: 'openssl torture FTP' install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON - tflags: '-t --shallow=20 FTP' + tflags: '-t --shallow=20 FTP --min=260' torture: true - name: 'openssl i686' @@ -215,6 +217,7 @@ jobs: --with-openssl --with-librtmp --with-libssh2 --with-libidn2 --enable-ares --enable-debug - name: '!ssl !http !smtp !imap' + tflags: '--min=475' configure: --without-ssl --enable-debug --disable-http --disable-smtp --disable-imap --disable-unity - name: 'libressl Fil-C' @@ -303,11 +306,12 @@ jobs: LDFLAGS: -fsanitize=memory LIBS: -ldl configure: CC=clang --without-ssl --without-zlib --without-brotli --without-zstd --without-libpsl --without-nghttp2 --enable-debug + tflags: '--min=1480' - name: 'event-based' install_packages: libssh-dev configure: --enable-debug --disable-shared --disable-threaded-resolver --with-libssh --with-openssl - tflags: '-n --test-event' + tflags: '-n --test-event --min=1350' - name: 'duphandle' install_packages: libssh-dev diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index cae51f5f7e..320689bda8 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -44,6 +44,7 @@ permissions: {} env: CURL_CI: github + CURL_TEST_MIN: 1700 MAKEFLAGS: -j 4 LDFLAGS: -w # suppress 'object file was built for newer macOS version than being linked' warnings @@ -225,16 +226,20 @@ jobs: compiler: clang install: brotli zstd configure: --without-ssl --with-brotli --with-zstd + tflags: '--min=1450' xcode: '' # default Xcode. Set it once to silence actionlint. - name: '!ssl !debug' compiler: gcc-13 configure: --without-ssl + tflags: '--min=950' - name: '!ssl' compiler: clang configure: --enable-debug --without-ssl + tflags: '--min=1500' - name: '!ssl libssh2 AppleIDN' compiler: clang configure: --enable-debug --with-libssh2=/opt/homebrew/opt/libssh2 --without-ssl --with-apple-idn + tflags: '--min=1550' - name: 'OpenSSL libssh c-ares' compiler: clang install: libssh @@ -246,8 +251,10 @@ jobs: - name: '!ssl c-ares' compiler: clang configure: --enable-debug --enable-ares --without-ssl + tflags: '--min=1500' - name: '!ssl HTTP-only' compiler: clang + tflags: '--min=950' configure: >- --enable-debug --disable-alt-svc --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap @@ -277,7 +284,7 @@ jobs: - name: 'OpenSSL event-based' compiler: clang configure: --enable-debug --with-openssl=/opt/homebrew/opt/openssl - tflags: --test-event + tflags: '--test-event --min=1300' # cmake - name: 'OpenSSL gsasl rtmp AppleIDN SecTrust +examples' install: libnghttp3 libngtcp2 gsasl rtmpdump @@ -328,18 +335,19 @@ jobs: compiler: clang install: rustls-ffi generate: -DENABLE_DEBUG=ON -DCURL_USE_RUSTLS=ON -DUSE_ECH=ON -DCURL_DISABLE_LDAP=ON + tflags: '--min=1650' - name: 'OpenSSL torture !FTP' compiler: clang install: libnghttp3 install_steps: torture generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON - tflags: -t --shallow=25 !FTP + tflags: '-t --shallow=25 !FTP --min=1450' - name: 'OpenSSL torture FTP' compiler: clang install: libnghttp3 install_steps: torture generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON - tflags: -t --shallow=20 FTP + tflags: '-t --shallow=20 FTP --min=260' exclude: # opt out jobs from combinations that have the compiler set manually - { compiler: llvm@18, build: { compiler: 'clang' } } diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml index d05a0791a7..282d561d78 100644 --- a/.github/workflows/non-native.yml +++ b/.github/workflows/non-native.yml @@ -37,6 +37,7 @@ permissions: {} env: CURL_CI: github + CURL_TEST_MIN: 1750 jobs: netbsd: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d3d3de8522..364c058fd7 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -37,6 +37,7 @@ permissions: {} env: CURL_CI: github + CURL_TEST_MIN: 1700 jobs: cygwin: @@ -197,26 +198,27 @@ jobs: matrix: include: # MSYS - - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' } - - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' } - - { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' } - - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' } + - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '--min=1550', config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' } + - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' } + - { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' } + - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' } # MinGW - - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' } - - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' } - - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' } - - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' } + - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' } + - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' } + - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '--min=1650', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' } + # MinGW torture + - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP --min=1300', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' } # WARNING: libssh uses hard-coded world-writable paths (/etc/..., ~/.ssh/) to # read its configuration from, making it vulnerable to attacks on # Windows. Do not use this component till there is a fix for these. # https://github.com/curl/curl-for-win/blob/3951808deb04df9489ee17430f236ed54436f81a/libssh.sh#L6-L8 - - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' } - - { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' } - - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' } - - { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } - # { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } - - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' } - - { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' } + - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' } + - { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' } + - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' } + - { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } + # { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } + - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' } + - { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' } fail-fast: false steps: - uses: msys2/setup-msys2@fb197b72ce45fb24f17bf3f807a388985654d1f2 # v2.29.0 @@ -415,6 +417,7 @@ jobs: run: shell: msys2 {0} env: + CURL_TEST_MIN: 1550 MAKEFLAGS: -j 5 MATRIX_DIR: '${{ matrix.dir }}' strategy: @@ -775,6 +778,7 @@ jobs: type: 'Debug' image: 'windows-11-arm' openssh: 'OpenSSH-Windows' + tflags: '--min=1650' config: >- -DENABLE_DEBUG=ON -DCURL_USE_SCHANNEL=ON diff --git a/docs/runtests.md b/docs/runtests.md index b449b3f411..bb64b412df 100644 --- a/docs/runtests.md +++ b/docs/runtests.md @@ -182,7 +182,12 @@ Lists all test case names. ## `-m=[seconds]` -Set timeout for curl commands in tests +Set timeout for curl commands in tests. + +## `--min=[count]` + +Set the minimum number of tests to run. This overrides the `CURL_TEST_MIN` +environment variable, if any. ## `-n` diff --git a/tests/globalconfig.pm b/tests/globalconfig.pm index 8d5105a36c..0c18a1ea6b 100644 --- a/tests/globalconfig.pm +++ b/tests/globalconfig.pm @@ -50,6 +50,7 @@ BEGIN { $LOCKDIR $LOGDIR $maxtime + $mintotal $memanalyze $MEMDUMP $perlcmd @@ -104,6 +105,7 @@ our $CURLVERSION=""; # curl's reported version number our $CURLVERNUM=""; # curl's reported version number (without -DEV) our $randseed = 0; # random number seed our $maxtime; # curl command timeout override +our $mintotal; # minimum number of tests to run # paths our $pwd = getcwd(); # current working directory diff --git a/tests/runtests.pl b/tests/runtests.pl index ef57dc3ec8..d1a1e40935 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -2403,6 +2403,10 @@ while(@ARGV) { my ($num)=($1); $maxtime=$num; } + elsif($ARGV[0] =~ /--min=(\d+)/) { + my ($num)=($1); + $mintotal=$num; + } elsif($ARGV[0] eq "-n") { # no valgrind undef $valgrind; @@ -2546,6 +2550,7 @@ Usage: runtests.pl [options] [test selection(s)] -L path require an additional perl library file to replace certain functions -l list all test case names/descriptions -m=[seconds] set timeout for curl commands in tests + --min=[count] minimum number of tests to run. -n no valgrind --no-debuginfod disable the valgrind debuginfod functionality -o variable=value set internal variable to the specified value @@ -3332,6 +3337,19 @@ else { } } +if(!$mintotal && $ENV{"CURL_TEST_MIN"}) { + $mintotal = $ENV{"CURL_TEST_MIN"}; +} +if($mintotal) { + if($total < $mintotal) { + logmsg "TESTFAIL: number of tests run was below the minimum of: $mintotal\n"; + exit 1; + } + else { + logmsg "TESTDONE: minimum number of tests was met: $mintotal\n"; + } +} + if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) { exit 1; }