thrdpool: retry failed thread starts while items wait

Verified in test 3306

Closes #22303

lib/asyn-thrdd.c

@@ -708,6 +708,15 @@ CURLcode Curl_async_pollset(struct Curl_easy *data,
     else
       stutter_ms = 200;
     timeout_ms = CURLMIN(stutter_ms, timeout_ms);
+#else
+    if(async->queries_ongoing &&
+       !Curl_thrdq_check_started(data->multi->resolv_thrdq)) {
+      /* The queue has items but starting a worker thread to process
+         them just failed again; expire soon to check once more,
+         instead of sleeping on the full resolve timeout. */
+      CURL_TRC_DNS(data, "resolver thread start failed again, retrying");
+      timeout_ms = CURLMIN(100, timeout_ms);
+    }
 #endif
     Curl_expire(data, timeout_ms, EXPIRE_ASYNC_NAME);
   }

lib/thrdpool.c

@@ -28,6 +28,7 @@
 #include "llist.h"
 #include "curl_threads.h"
 #include "curlx/timeval.h"
+#include "curlx/strparse.h"
 #include "thrdpool.h"
 #ifdef CURLVERBOSE
 #include "curl_trc.h"
@@ -64,6 +65,9 @@ struct curl_thrdpool {
   uint32_t max_threads;
   uint32_t idle_time_ms;
   uint32_t next_id;
+#ifdef DEBUGBUILD
+  int dbg_fail_starts; /* fail this many thread starts */
+#endif
   BIT(aborted);
   BIT(detached);
 };
@@ -166,7 +170,14 @@ static CURLcode thrdslot_start(struct curl_thrdpool *tpool)
 
   tpool->refcount++;
   tslot->running = TRUE;
-  tslot->thread = Curl_thread_create(thrdslot_run, tslot);
+#ifdef DEBUGBUILD
+  if(tpool->dbg_fail_starts > 0) {
+    --tpool->dbg_fail_starts;
+    tslot->thread = curl_thread_t_null;
+  }
+  else
+#endif
+    tslot->thread = Curl_thread_create(thrdslot_run, tslot);
   if(tslot->thread == curl_thread_t_null) { /* never started */
     tslot->running = FALSE;
     thrdpool_unlink(tpool, TRUE);
@@ -320,6 +331,17 @@ CURLcode Curl_thrdpool_create(struct curl_thrdpool **ptpool,
   if(!tpool->name)
     goto out;
 
+#ifdef DEBUGBUILD
+  {
+    const char *p = getenv("CURL_DBG_THRDPOOL_FAIL_STARTS");
+    if(p) {
+      curl_off_t l;
+      if(!curlx_str_number(&p, &l, INT_MAX))
+        tpool->dbg_fail_starts = (int)l;
+    }
+  }
+#endif
+
   result = Curl_thrdpool_set_props(tpool, min_threads, max_threads,
                                    idle_time_ms);
 

lib/thrdqueue.c

@@ -295,10 +295,22 @@ out:
   return result;
 }
 
+bool Curl_thrdq_check_started(struct curl_thrdq *tqueue)
+{
+  size_t unprocessed;
+
+  Curl_mutex_acquire(&tqueue->lock);
+  unprocessed = tqueue->aborted ? 0 : Curl_llist_count(&tqueue->sendq);
+  Curl_mutex_release(&tqueue->lock);
+  return !unprocessed ||
+         !Curl_thrdpool_signal(tqueue->tpool, (uint32_t)unprocessed);
+}
+
 CURLcode Curl_thrdq_recv(struct curl_thrdq *tqueue, void **pitem)
 {
   CURLcode result = CURLE_AGAIN;
   struct Curl_llist_node *e;
+  size_t signals = 0;
 
   *pitem = NULL;
   Curl_mutex_acquire(&tqueue->lock);
@@ -316,8 +328,17 @@ CURLcode Curl_thrdq_recv(struct curl_thrdq *tqueue, void **pitem)
     thrdq_item_destroy(qitem);
     result = CURLE_OK;
   }
+  else
+    signals = Curl_llist_count(&tqueue->sendq);
 out:
   Curl_mutex_release(&tqueue->lock);
+  /* Signal thread pool unlocked to avoid deadlocks. If items await
+   * processing while nothing was ready, make sure the pool has a
+   * thread to work on them. An earlier thread start may have failed,
+   * which `Curl_thrdq_send()` cannot report to its caller. Without
+   * this, such items would sit unprocessed until the next send. */
+  if(signals)
+    (void)Curl_thrdpool_signal(tqueue->tpool, (uint32_t)signals);
   return result;
 }
 

lib/thrdqueue.h

@@ -86,10 +86,21 @@ CURLcode Curl_thrdq_send(struct curl_thrdq *tqueue, void *item,
  * The caller takes ownership of the item received, e.g. the queue
  * relinquishes all references to item.
  * Returns CURLE_AGAIN when there is no processed item, setting `pitem`
- * to NULL.
+ * to NULL. When nothing has been processed while items await sending,
+ * the pool is signalled to make sure a worker thread exists: an
+ * earlier thread start may have failed.
  */
 CURLcode Curl_thrdq_recv(struct curl_thrdq *tqueue, void **pitem);
 
+/* Check that items awaiting processing have a worker thread to run
+ * them, signalling the pool to start one when needed -- an earlier
+ * thread start may have failed. Returns TRUE when everything is ok:
+ * no items are waiting or the pool took the signal. FALSE when a
+ * thread start just failed again; callers may want to check again
+ * soon rather than wait indefinitely.
+ */
+bool Curl_thrdq_check_started(struct curl_thrdq *tqueue);
+
 /* Return TRUE if the passed "item" matches. */
 typedef bool Curl_thrdq_item_match_cb(void *item, void *match_data);