runtests: add retry option to reduce flakiness

Add `--retry=<num>` option to tell runtests to retry the first `<num>`
tests that failed. Retries aren't run right away, but added to the end
of the test queue. Once all retry slots are used, test fail as normal.

In CI, typically a single test fails for flakiness, and rarely over 5.

Make the `ci-test` targets default to `--retry=5`.

Closes #17091

tests/CMakeLists.txt

@@ -103,7 +103,7 @@ curl_add_runtests(test-am        "-a -am")
 curl_add_runtests(test-full      "-a -p -r")
 # ~flaky means that it ignores results of tests using the flaky keyword
 curl_add_runtests(test-nonflaky  "-a -p ~flaky ~timing-dependent")
-curl_add_runtests(test-ci        "-a -p ~flaky ~timing-dependent -r -j20")
+curl_add_runtests(test-ci        "-a -p ~flaky ~timing-dependent -r --retry=5 -j20")
 curl_add_runtests(test-torture   "-a -t -j20")
 curl_add_runtests(test-event     "-a -e")
 

tests/Makefile.am

@@ -142,7 +142,7 @@ TEST_E = -a -e
 TEST_NF = -a -p ~flaky ~timing-dependent
 
 # special CI target derived from nonflaky with CI-specific flags
-TEST_CI = $(TEST_NF) -r -j20
+TEST_CI = $(TEST_NF) -r --retry=5 -j20
 
 PYTEST = pytest
 endif

tests/runtests.md

@@ -228,6 +228,10 @@ sequence at the end of the initially given one.
 If **-R** option is also used, the scrambling is done after the repeats have
 extended the test sequence.
 
+## `--retry=[num]`
+
+Number of attempts for the whole test run to retry failed tests.
+
 ## `-s`
 
 Shorter output. Speaks less than default.

tests/runtests.pl

@@ -123,6 +123,7 @@ my $TESTCASES="all";
 
 my $libtool;
 my $repeat = 0;
+my $retry = 0;
 
 my $start;          # time at which testing started
 my $args;           # command-line arguments
@@ -2353,6 +2354,10 @@ while(@ARGV) {
         # Repeat-run the given tests this many times
         $repeat = $1;
     }
+    elsif($ARGV[0] =~ /--retry=(\d+)/) {
+        # Number of attempts for the whole test run to retry failed tests
+        $retry = $1;
+    }
     elsif($ARGV[0] =~ /--seed=(\d+)/) {
         # Set a fixed random seed (used for -R and --shallow)
         $randseed = $1;
@@ -2461,6 +2466,7 @@ Usage: runtests.pl [options] [test selection(s)]
   -r       run time statistics
   -rf      full run time statistics
   --repeat=[num] run the given tests this many times
+  --retry=[num] number of attempts for the whole test run to retry failed tests
   -s       short output
   --seed=[num] set the random seed to a fixed number
   --shallow=[num] randomly makes the torture tests "thinner"
@@ -2875,9 +2881,12 @@ sub displaylogs {
 
 my $failed;
 my $failedign;
+my $failedre;
 my $ok=0;
 my $ign=0;
 my $total=0;
+my $executed=0;
+my $retry_done=0;
 my $lasttest=0;
 my @at = split(" ", $TESTCASES);
 my $count=0;
@@ -2925,6 +2934,16 @@ createrunners($numrunners);
 
 # run through each candidate test and execute it
 my $runner_wait_cnt = 0;
+
+# number of retry attempts for the whole test run
+my $retry_left;
+if($torture) {
+    $retry_left = 0;  # No use of retrying torture tests
+}
+else {
+    $retry_left = $retry;
+}
+
 while () {
     # check the abort flag
     if($globalabort) {
@@ -3014,6 +3033,7 @@ while () {
                     }
 
                     $total++; # number of tests we've run
+                    $executed++;
 
                     if($error>0) {
                         if($error==2) {
@@ -3021,7 +3041,17 @@ while () {
                             $failedign .= "$testnum ";
                         }
                         else {
-                            $failed.= "$testnum ";
+                            # make another attempt to counteract flaky failures
+                            if($retry_left > 0) {
+                                $retry_left--;
+                                $retry_done++;
+                                $total--;
+                                push(@runtests, $testnum);
+                                $failedre .= "$testnum ";
+                            }
+                            else {
+                                $failed.= "$testnum ";
+                            }
                         }
                         if($postmortem) {
                             # display all files in $LOGDIR/ in a nice way
@@ -3176,7 +3206,15 @@ sub testnumdetails {
     }
 }
 
-if($total) {
+if($executed) {
+    if($failedre) {
+        my $sorted = numsortwords($failedre);
+        logmsg "::group::Failed Retried Test details\n";
+        testnumdetails("FAIL-RETRIED", $sorted);
+        logmsg "RETRIED: failed tests: $sorted\n";
+        logmsg "::endgroup::\n";
+    }
+
     if($passedign) {
         my $sorted = numsortwords($passedign);
         logmsg "::group::Passed Ignored Test details\n";