multi: add new information extraction method

Adds `curl_off_t curl_multi_get_offt(CURLM *multi_handle, CURLMinfo_offt info)`
to the multi interface with enums:

* CURLMINFO_XFERS_CURRENT: current number of transfers
* CURLMINFO_XFERS_RUNNING: number of running transfers
* CURLMINFO_XFERS_PENDING: number of pending transfers
* CURLMINFO_XFERS_DONE: number of finished transfers to read
* CURLMINFO_XFERS_ADDED: total number of transfers added, ever

Add documentation for functions and info enums.

Add use in the curl command line tool to replace two static
variables counting the same "from the outside".

refs #17870

.github/scripts/spellcheck.curl

@@ -132,6 +132,7 @@ curl_multi_timeout
 curl_multi_setopt
 curl_multi_assign
 curl_multi_get_handles
+curl_multi_get_offt
 curl_pushheader_bynum
 curl_pushheader_byname
 curl_multi_waitfds

docs/libcurl/Makefile.inc

@@ -75,6 +75,7 @@ man_MANS = \
  curl_multi_cleanup.3 \
  curl_multi_fdset.3 \
  curl_multi_get_handles.3 \
+ curl_multi_get_offt.3 \
  curl_multi_info_read.3 \
  curl_multi_init.3 \
  curl_multi_perform.3 \

docs/libcurl/curl_multi_get_offt.md

@@ -0,0 +1,97 @@
+---
+c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+SPDX-License-Identifier: curl
+Title: curl_multi_get_offt
+Section: 3
+Source: libcurl
+See-also:
+  - curl_multi_add_handle (3)
+  - curl_multi_remove_handle (3)
+Protocol:
+  - All
+Added-in: 8.16.0
+---
+
+# NAME
+
+curl_multi_get_offt - extract information from a multi handle
+
+# SYNOPSIS
+
+~~~c
+#include <curl/curl.h>
+
+curl_off_t curl_multi_get_offt(CURLM *multi, CURLMinfo info);
+~~~
+
+# DESCRIPTION
+
+Get the *info* kept in the *multi* handle for `CURLMI_OFFT_*`.
+If the multi handle is not valid or the *info* is not applicable, returns 0.
+
+# OPTIONS
+
+The following information can be extracted:
+
+## CURLMINFO_XFERS_CURRENT
+
+The number of easy handles currently added to the multi. This does not
+count handles removed. It does count internal handles that get
+added for tasks (like resolving via DoH, for example).
+
+For the total number of easy handles ever added to the multi, see
+curl_multi_get_offt(3).
+
+## CURLMINFO_XFERS_RUNNING
+
+The number of easy handles currently running, e.g. where the transfer
+has started but not finished yet.
+
+## CURLMINFO_XFERS_PENDING
+
+The number of current easy handles waiting to start. An added transfer
+might become pending for various reasons: a connection limit forces it
+to wait, resolving DNS is not finished or it is not clear if an existing,
+matching connection may allow multiplexing (HTTP/2 or HTTP/3).
+
+## CURLMINFO_XFERS_DONE
+
+The number of easy handles currently finished, but not yet processed
+via curl_multi_info_read(3).
+
+## CURLMINFO_XFERS_ADDED
+
+The cumulative number of all easy handles added to the multi, ever.
+This includes internal handles added for tasks (like resolving
+via DoH, for example).
+
+For the current number of easy handles managed by the multi, use
+*CURLMINFO_XFERS_CURRENT*.
+
+# %PROTOCOLS%
+
+# EXAMPLE
+
+~~~c
+int main(void)
+{
+  /* init a multi stack */
+  CURLM *multi = curl_multi_init();
+  CURL *curl = curl_easy_init();
+  curl_off_t n;
+
+  if(curl) {
+    /* add the transfer */
+    curl_multi_add_handle(multi, curl);
+
+    n = curl_multi_get_offt(multi, CURLMI_OFFT_XTOTAL);
+    /* on successful add, n is 1 */
+  }
+}
+~~~
+
+# %AVAILABILITY%
+
+# RETURN VALUE
+
+The extracted value

docs/libcurl/symbols-in-versions

@@ -553,6 +553,12 @@ CURLM_RECURSIVE_API_CALL        7.59.0
 CURLM_UNKNOWN_OPTION            7.15.4
 CURLM_UNRECOVERABLE_POLL        7.84.0
 CURLM_WAKEUP_FAILURE            7.68.0
+CURLMINFO_NONE                  8.16.0
+CURLMINFO_XFERS_CURRENT         8.16.0
+CURLMINFO_XFERS_RUNNING         8.16.0
+CURLMINFO_XFERS_PENDING         8.16.0
+CURLMINFO_XFERS_DONE            8.16.0
+CURLMINFO_XFERS_ADDED           8.16.0
 CURLMIMEOPT_FORMESCAPE          7.81.0
 CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE 7.30.0
 CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE 7.30.0

include/curl/multi.h

@@ -448,6 +448,36 @@ CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle,
  */
 CURL_EXTERN CURL **curl_multi_get_handles(CURLM *multi_handle);
 
+
+typedef enum {
+  CURLMINFO_NONE, /* first, never use this */
+  /* The number of easy handles currently managed by the multi handle,
+   * e.g. have been added but not yet removed. */
+  CURLMINFO_XFERS_CURRENT   = 1,
+  /* The number of easy handles running, e.g. not done and not queueing. */
+  CURLMINFO_XFERS_RUNNING = 2,
+  /* The number of easy handles waiting to start, e.g. for a connection
+   * to become available due to limits on parallelism, max connections
+   * or other factors. */
+  CURLMINFO_XFERS_PENDING = 3,
+  /* The number of easy handles finished, waiting for their results to
+   * be read via `curl_multi_info_read()`. */
+  CURLMINFO_XFERS_DONE = 4,
+  /* The total number of easy handles added to the multi handle, ever. */
+  CURLMINFO_XFERS_ADDED = 5
+} CURLMinfo_offt;
+
+/*
+ * Name:    curl_multi_get_offt()
+ *
+ * Desc:    Retrieves a numeric value for the `CURLMINFO_*` enums.
+ *
+ * Returns: the curl_off_t value from a valid multi handle and
+ *          supported `info` or -1.
+ */
+CURL_EXTERN curl_off_t curl_multi_get_offt(CURLM *multi_handle,
+                                           CURLMinfo_offt info);
+
 /*
  * Name: curl_push_callback
  *

lib/libcurl.def

@@ -54,6 +54,7 @@ curl_multi_assign
 curl_multi_cleanup
 curl_multi_fdset
 curl_multi_get_handles
+curl_multi_get_offt
 curl_multi_info_read
 curl_multi_init
 curl_multi_perform

lib/multi.c

@@ -479,6 +479,7 @@ CURLMcode curl_multi_add_handle(CURLM *m, CURL *d)
   /* add the easy handle to the process set */
   Curl_uint_bset_add(&multi->process, data->mid);
   ++multi->xfers_alive;
+  ++multi->xfers_total_ever;
 
   Curl_cpool_xfer_init(data);
   multi_warn_debug(multi, data);
@@ -3734,6 +3735,33 @@ CURL **curl_multi_get_handles(CURLM *m)
   return a;
 }
 
+CURL_EXTERN curl_off_t curl_multi_get_offt(CURLM *m, CURLMinfo_offt info)
+{
+  struct Curl_multi *multi = m;
+
+  if(!GOOD_MULTI_HANDLE(multi))
+    return -1;
+
+  switch(info) {
+  case CURLMINFO_XFERS_CURRENT: {
+    unsigned int n = Curl_uint_tbl_count(&multi->xfers);
+    if(n && multi->admin)
+      --n;
+    return (curl_off_t)n;
+  }
+  case CURLMINFO_XFERS_RUNNING:
+    return (curl_off_t)Curl_uint_bset_count(&multi->process);
+  case CURLMINFO_XFERS_PENDING:
+    return (curl_off_t)Curl_uint_bset_count(&multi->pending);
+  case CURLMINFO_XFERS_DONE:
+    return (curl_off_t)Curl_uint_bset_count(&multi->msgsent);
+  case CURLMINFO_XFERS_ADDED:
+    return multi->xfers_total_ever;
+  default:
+    return -1;
+  }
+}
+
 CURLcode Curl_multi_xfer_buf_borrow(struct Curl_easy *data,
                                     char **pbuf, size_t *pbuflen)
 {

lib/multihandle.h

@@ -95,6 +95,7 @@ struct Curl_multi {
 
   unsigned int xfers_alive; /* amount of added transfers that have
                                not yet reached COMPLETE state */
+  curl_off_t xfers_total_ever; /* total of added transfers, ever. */
   struct uint_tbl xfers; /* transfers added to this multi */
   /* Each transfer's mid may be present in at most one of these */
   struct uint_bset process; /* transfer being processed */

lib/uint-bset.c

@@ -79,8 +79,9 @@ UNITTEST unsigned int Curl_uint_bset_capacity(struct uint_bset *bset)
 {
   return bset->nslots * 64;
 }
+#endif
 
-UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset)
+unsigned int Curl_uint_bset_count(struct uint_bset *bset)
 {
   unsigned int i;
   unsigned int n = 0;
@@ -90,7 +91,6 @@ UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset)
   }
   return n;
 }
-#endif
 
 bool Curl_uint_bset_empty(struct uint_bset *bset)
 {

scripts/singleuse.pl

@@ -111,6 +111,7 @@ my %api = (
     'curl_multi_cleanup' => 'API',
     'curl_multi_fdset' => 'API',
     'curl_multi_get_handles' => 'API',
+    'curl_multi_get_offt' => 'API',
     'curl_multi_info_read' => 'API',
     'curl_multi_init' => 'API',
     'curl_multi_perform' => 'API',

src/tool_operate.c

@@ -207,7 +207,6 @@ static curl_off_t VmsSpecialSize(const char *name,
 
 struct per_transfer *transfers; /* first node */
 static struct per_transfer *transfersl; /* last node */
-static curl_off_t all_pers;
 
 /* add_per_transfer creates a new 'per_transfer' node in the linked
    list of transfers */
@@ -229,8 +228,6 @@ static CURLcode add_per_transfer(struct per_transfer **per)
     transfersl = p;
   }
   *per = p;
-  all_xfers++; /* count total number of transfers added */
-  all_pers++;
 
   return CURLE_OK;
 }
@@ -259,7 +256,6 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per)
     transfersl = p;
 
   free(per);
-  all_pers--;
 
   return n;
 }
@@ -1422,7 +1418,8 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global,
   char *errorbuf;
   *addedp = FALSE;
   *morep = FALSE;
-  if(all_pers < (global->parallel_max*2)) {
+  if(curl_multi_get_offt(multi, CURLMINFO_XFERS_CURRENT) <
+     (curl_off_t)(global->parallel_max*2)) {
     bool skipped = FALSE;
     do {
       result = create_transfer(global, share, addedp, &skipped);
@@ -1760,7 +1757,7 @@ static CURLcode check_finished(struct parastate *s)
   CURLMsg *msg;
   bool checkmore = FALSE;
   struct GlobalConfig *global = s->global;
-  progress_meter(global, &s->start, FALSE);
+  progress_meter(global, s->multi, &s->start, FALSE);
   do {
     msg = curl_multi_info_read(s->multi, &rc);
     if(msg) {
@@ -1885,7 +1882,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
         result = check_finished(s);
     }
 
-    (void)progress_meter(global, &s->start, TRUE);
+    (void)progress_meter(global, s->multi, &s->start, TRUE);
   }
 
   /* Make sure to return some kind of error if there was a multi problem */

src/tool_progress.c

@@ -134,8 +134,6 @@ static curl_off_t all_ultotal = 0;
 static curl_off_t all_dlalready = 0;
 static curl_off_t all_ulalready = 0;
 
-curl_off_t all_xfers = 0;   /* current total */
-
 struct speedcount {
   curl_off_t dl;
   curl_off_t ul;
@@ -151,6 +149,7 @@ static struct speedcount speedstore[SPEEDCNT];
   |  6 --   9.9G     0     2     2   0:00:40  0:00:02  0:00:37 4087M
 */
 bool progress_meter(struct GlobalConfig *global,
+                    CURLM *multi,
                     struct curltime *start,
                     bool final)
 {
@@ -184,7 +183,6 @@ bool progress_meter(struct GlobalConfig *global,
     curl_off_t all_ulnow = 0;
     bool dlknown = TRUE;
     bool ulknown = TRUE;
-    curl_off_t all_running = 0; /* in progress */
     curl_off_t speed = 0;
     unsigned int i;
     stamp = now;
@@ -210,8 +208,6 @@ bool progress_meter(struct GlobalConfig *global,
         all_ultotal += per->ultotal;
         per->ultotal_added = TRUE;
       }
-      if(per->added)
-        all_running++;
     }
     if(dlknown && all_dltotal)
       msnprintf(dlpercen, sizeof(dlpercen), "%3" CURL_FORMAT_CURL_OFF_T,
@@ -292,8 +288,8 @@ bool progress_meter(struct GlobalConfig *global,
             ulpercen,  /* 3 letters */
             max5data(all_dlnow, buffer[0]),
             max5data(all_ulnow, buffer[1]),
-            all_xfers,
-            all_running,
+            curl_multi_get_offt(multi, CURLMINFO_XFERS_ADDED),
+            curl_multi_get_offt(multi, CURLMINFO_XFERS_RUNNING),
             time_total,
             time_spent,
             time_left,

src/tool_progress.h

@@ -32,10 +32,9 @@ int xferinfo_cb(void *clientp,
                 curl_off_t ulnow);
 
 bool progress_meter(struct GlobalConfig *global,
+                    CURLM *multi,
                     struct curltime *start,
                     bool final);
 void progress_finalize(struct per_transfer *per);
 
-extern curl_off_t all_xfers;   /* total number */
-
 #endif /* HEADER_CURL_TOOL_PROGRESS_H */

tests/data/test1135

@@ -109,6 +109,7 @@ curl_multi_timeout
 curl_multi_setopt
 curl_multi_assign
 curl_multi_get_handles
+curl_multi_get_offt
 curl_pushheader_bynum
 curl_pushheader_byname
 curl_multi_waitfds