thrdpool: retry failed thread starts while items wait

Verified in test 3306

Closes #22303

tests/data/Makefile.am

@@ -286,7 +286,7 @@ test3208 test3209 test3210 test3211 test3212 test3213 test3214 test3215 \
 test3216 test3217 test3218 test3219 test3220 test3221 test3222 \
 test3223 test3224 test3225 test3226 test3227 test3228 test3229 \
 \
-test3300 test3301 test3302 test3303 test3304 test3305 \
+test3300 test3301 test3302 test3303 test3304 test3305 test3306 \
 \
 test3400 test3401 \
 \

tests/data/test3306

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<testcase>
+<info>
+<keywords>
+unittest
+threads
+</keywords>
+</info>
+
+# Client-side
+<client>
+<features>
+unittest
+Debug
+</features>
+<setenv>
+CURL_DBG_THRDPOOL_FAIL_STARTS=5
+</setenv>
+<name>
+thrdqueue retries failed thread starts on receive
+</name>
+</client>
+</testcase>

tests/http/test_21_resolve.py

@@ -265,6 +265,24 @@ class TestResolve:
         if env.curl_is_verbose():
             assert not [t for t in r.trace_lines if 'Negative DNS entry' in t], f'{r}'
 
+    # a resolve gets processed even when the first resolver thread
+    # starts fail, e.g. when the system temporarily refuses to spawn
+    # threads. The transfer must fail on the (debug-forced) lookup
+    # failure well before the resolve timeout.
+    @pytest.mark.skipif(condition=not Env.curl_resolv_threaded(), reason="no threaded resolver")
+    def test_21_15_resolv_thread_start_fails(self, env: Env, httpd, nghttpx):
+        run_env = os.environ.copy()
+        run_env['CURL_DBG_THRDPOOL_FAIL_STARTS'] = '3'
+        run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = '10'
+        curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
+        url = 'https://test-1.http.curl.invalid/'
+        r = curl.http_download(urls=[url], with_stats=True, extra_args=[
+            '--connect-timeout', '20'
+        ])
+        r.check_exit_code(6)
+        r.check_stats(count=1, http_status=0, exitcode=6)
+        assert r.duration < timedelta(seconds=20), f'{r}'
+
     def _clean_files(self, files):
         for file in files:
             if os.path.exists(file):

tests/unit/Makefile.inc

@@ -50,4 +50,5 @@ TESTS_C = \
   unit3200.c                                             unit3205.c \
   unit3211.c unit3212.c unit3213.c unit3214.c            unit3216.c unit3219.c \
   unit3227.c \
-  unit3300.c unit3301.c unit3302.c unit3303.c unit3304.c unit3400.c
+  unit3300.c unit3301.c unit3302.c unit3303.c unit3304.c unit3306.c \
+  unit3400.c

tests/unit/unit3306.c

@@ -0,0 +1,120 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * SPDX-License-Identifier: curl
+ *
+ ***************************************************************************/
+#include "unitcheck.h"
+
+#include "curlx/wait.h"
+#include "thrdqueue.h"
+#include "curl_threads.h"
+
+#if defined(USE_THREADS) && defined(DEBUGBUILD)
+
+struct unit3306_item {
+  int id;
+  BIT(processed);
+};
+
+static struct unit3306_item *unit3306_item_create(int id)
+{
+  struct unit3306_item *uitem;
+  uitem = curlx_calloc(1, sizeof(*uitem));
+  if(uitem) {
+    uitem->id = id;
+    curl_mfprintf(stderr, "created item %d\n", uitem->id);
+  }
+  return uitem;
+}
+
+static void unit3306_item_free(void *item)
+{
+  struct unit3306_item *uitem = item;
+  curl_mfprintf(stderr, "free item %d\n", uitem->id);
+  curlx_free(uitem);
+}
+
+static void unit3306_process(void *item)
+{
+  struct unit3306_item *uitem = item;
+  curlx_wait_ms(1);
+  uitem->processed = TRUE;
+}
+
+/* The test runs with CURL_DBG_THRDPOOL_FAIL_STARTS=5 set, making the
+ * pool's first 5 thread starts fail, as if the system temporarily ran
+ * against a thread limit. Each of the 3 sends attempts one thread
+ * start (3 failures) and no thread exists to process the queue. Only
+ * the receive side signalling the pool again - consuming the
+ * remaining 2 armed failures, one per empty receive, and then
+ * starting workers - lets the items get processed. */
+static CURLcode test_unit3306(const char *arg)
+{
+  UNITTEST_BEGIN_SIMPLE
+  struct curl_thrdq *tqueue;
+  int i, count = 3, nrecvd;
+  CURLcode result;
+
+  fail_unless(getenv("CURL_DBG_THRDPOOL_FAIL_STARTS"),
+              "CURL_DBG_THRDPOOL_FAIL_STARTS must be set for this test");
+
+  result = Curl_thrdq_create(&tqueue, "unit3306", 0, 0, 2, 1,
+                             unit3306_item_free, unit3306_process,
+                             NULL, NULL);
+  fail_unless(!result, "queue create");
+  for(i = 0; i < count; ++i) {
+    struct unit3306_item *uitem = unit3306_item_create(i);
+    fail_unless(uitem, "item create");
+    result = Curl_thrdq_send(tqueue, uitem, NULL, 0);
+    fail_unless(!result, "send");
+  }
+
+  /* no send was able to start a thread. Receive until all items come
+     back processed, which needs the recv side to signal the pool. */
+  nrecvd = 0;
+  for(i = 0; (nrecvd < count) && (i < 10000); ++i) {
+    void *item;
+    result = Curl_thrdq_recv(tqueue, &item);
+    fail_unless(!result || (result == CURLE_AGAIN), "recv");
+    if(item) {
+      struct unit3306_item *uitem = item;
+      curl_mfprintf(stderr, "received item %d\n", uitem->id);
+      ++nrecvd;
+      fail_unless(uitem->processed, "recv unprocessed item");
+      unit3306_item_free(item);
+    }
+    else
+      curlx_wait_ms(1);
+  }
+  Curl_thrdq_destroy(tqueue, TRUE);
+  tqueue = NULL;
+  fail_unless(nrecvd == count, "items not processed");
+
+  UNITTEST_END_SIMPLE
+}
+
+#else
+static CURLcode test_unit3306(const char *arg)
+{
+  UNITTEST_BEGIN_SIMPLE
+  UNITTEST_END_SIMPLE
+}
+#endif /* USE_THREADS && DEBUGBUILD */