From 923055aed67c56940f4a46390c23eda1f5396b66 Mon Sep 17 00:00:00 2001 From: Stefan Eissing Date: Wed, 23 Jul 2025 09:18:59 +0200 Subject: [PATCH] multi: add new information extraction method Adds `curl_off_t curl_multi_get_offt(CURLM *multi_handle, CURLMinfo_offt info)` to the multi interface with enums: * CURLMINFO_XFERS_CURRENT: current number of transfers * CURLMINFO_XFERS_RUNNING: number of running transfers * CURLMINFO_XFERS_PENDING: number of pending transfers * CURLMINFO_XFERS_DONE: number of finished transfers to read * CURLMINFO_XFERS_ADDED: total number of transfers added, ever Add documentation for functions and info enums. Add use in the curl command line tool to replace two static variables counting the same "from the outside". refs #17870 --- .github/scripts/spellcheck.curl | 1 + docs/libcurl/Makefile.inc | 1 + docs/libcurl/curl_multi_get_offt.md | 97 +++++++++++++++++++++++++++++ docs/libcurl/symbols-in-versions | 6 ++ include/curl/multi.h | 30 +++++++++ lib/libcurl.def | 1 + lib/multi.c | 28 +++++++++ lib/multihandle.h | 1 + lib/uint-bset.c | 4 +- scripts/singleuse.pl | 1 + src/tool_operate.c | 11 ++-- src/tool_progress.c | 10 +-- src/tool_progress.h | 3 +- tests/data/test1135 | 1 + 14 files changed, 177 insertions(+), 18 deletions(-) create mode 100644 docs/libcurl/curl_multi_get_offt.md diff --git a/.github/scripts/spellcheck.curl b/.github/scripts/spellcheck.curl index 4de9d86596..c24edf2b3b 100644 --- a/.github/scripts/spellcheck.curl +++ b/.github/scripts/spellcheck.curl @@ -132,6 +132,7 @@ curl_multi_timeout curl_multi_setopt curl_multi_assign curl_multi_get_handles +curl_multi_get_offt curl_pushheader_bynum curl_pushheader_byname curl_multi_waitfds diff --git a/docs/libcurl/Makefile.inc b/docs/libcurl/Makefile.inc index 83357cf42b..9142f65fab 100644 --- a/docs/libcurl/Makefile.inc +++ b/docs/libcurl/Makefile.inc @@ -75,6 +75,7 @@ man_MANS = \ curl_multi_cleanup.3 \ curl_multi_fdset.3 \ curl_multi_get_handles.3 \ + curl_multi_get_offt.3 \ curl_multi_info_read.3 \ curl_multi_init.3 \ curl_multi_perform.3 \ diff --git a/docs/libcurl/curl_multi_get_offt.md b/docs/libcurl/curl_multi_get_offt.md new file mode 100644 index 0000000000..3c9294ce4c --- /dev/null +++ b/docs/libcurl/curl_multi_get_offt.md @@ -0,0 +1,97 @@ +--- +c: Copyright (C) Daniel Stenberg, , et al. +SPDX-License-Identifier: curl +Title: curl_multi_get_offt +Section: 3 +Source: libcurl +See-also: + - curl_multi_add_handle (3) + - curl_multi_remove_handle (3) +Protocol: + - All +Added-in: 8.16.0 +--- + +# NAME + +curl_multi_get_offt - extract information from a multi handle + +# SYNOPSIS + +~~~c +#include + +curl_off_t curl_multi_get_offt(CURLM *multi, CURLMinfo info); +~~~ + +# DESCRIPTION + +Get the *info* kept in the *multi* handle for `CURLMI_OFFT_*`. +If the multi handle is not valid or the *info* is not applicable, returns 0. + +# OPTIONS + +The following information can be extracted: + +## CURLMINFO_XFERS_CURRENT + +The number of easy handles currently added to the multi. This does not +count handles removed. It does count internal handles that get +added for tasks (like resolving via DoH, for example). + +For the total number of easy handles ever added to the multi, see +curl_multi_get_offt(3). + +## CURLMINFO_XFERS_RUNNING + +The number of easy handles currently running, e.g. where the transfer +has started but not finished yet. + +## CURLMINFO_XFERS_PENDING + +The number of current easy handles waiting to start. An added transfer +might become pending for various reasons: a connection limit forces it +to wait, resolving DNS is not finished or it is not clear if an existing, +matching connection may allow multiplexing (HTTP/2 or HTTP/3). + +## CURLMINFO_XFERS_DONE + +The number of easy handles currently finished, but not yet processed +via curl_multi_info_read(3). + +## CURLMINFO_XFERS_ADDED + +The cumulative number of all easy handles added to the multi, ever. +This includes internal handles added for tasks (like resolving +via DoH, for example). + +For the current number of easy handles managed by the multi, use +*CURLMINFO_XFERS_CURRENT*. + +# %PROTOCOLS% + +# EXAMPLE + +~~~c +int main(void) +{ + /* init a multi stack */ + CURLM *multi = curl_multi_init(); + CURL *curl = curl_easy_init(); + curl_off_t n; + + if(curl) { + /* add the transfer */ + curl_multi_add_handle(multi, curl); + + n = curl_multi_get_offt(multi, CURLMI_OFFT_XTOTAL); + /* on successful add, n is 1 */ + } +} +~~~ + +# %AVAILABILITY% + +# RETURN VALUE + +The extracted value diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index 6387ea3014..d2a4cc598e 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -553,6 +553,12 @@ CURLM_RECURSIVE_API_CALL 7.59.0 CURLM_UNKNOWN_OPTION 7.15.4 CURLM_UNRECOVERABLE_POLL 7.84.0 CURLM_WAKEUP_FAILURE 7.68.0 +CURLMINFO_NONE 8.16.0 +CURLMINFO_XFERS_CURRENT 8.16.0 +CURLMINFO_XFERS_RUNNING 8.16.0 +CURLMINFO_XFERS_PENDING 8.16.0 +CURLMINFO_XFERS_DONE 8.16.0 +CURLMINFO_XFERS_ADDED 8.16.0 CURLMIMEOPT_FORMESCAPE 7.81.0 CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE 7.30.0 CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE 7.30.0 diff --git a/include/curl/multi.h b/include/curl/multi.h index 0fbea88707..ab4e8cd81a 100644 --- a/include/curl/multi.h +++ b/include/curl/multi.h @@ -448,6 +448,36 @@ CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle, */ CURL_EXTERN CURL **curl_multi_get_handles(CURLM *multi_handle); + +typedef enum { + CURLMINFO_NONE, /* first, never use this */ + /* The number of easy handles currently managed by the multi handle, + * e.g. have been added but not yet removed. */ + CURLMINFO_XFERS_CURRENT = 1, + /* The number of easy handles running, e.g. not done and not queueing. */ + CURLMINFO_XFERS_RUNNING = 2, + /* The number of easy handles waiting to start, e.g. for a connection + * to become available due to limits on parallelism, max connections + * or other factors. */ + CURLMINFO_XFERS_PENDING = 3, + /* The number of easy handles finished, waiting for their results to + * be read via `curl_multi_info_read()`. */ + CURLMINFO_XFERS_DONE = 4, + /* The total number of easy handles added to the multi handle, ever. */ + CURLMINFO_XFERS_ADDED = 5 +} CURLMinfo_offt; + +/* + * Name: curl_multi_get_offt() + * + * Desc: Retrieves a numeric value for the `CURLMINFO_*` enums. + * + * Returns: the curl_off_t value from a valid multi handle and + * supported `info` or -1. + */ +CURL_EXTERN curl_off_t curl_multi_get_offt(CURLM *multi_handle, + CURLMinfo_offt info); + /* * Name: curl_push_callback * diff --git a/lib/libcurl.def b/lib/libcurl.def index 43e26f655c..ae64776dc2 100644 --- a/lib/libcurl.def +++ b/lib/libcurl.def @@ -54,6 +54,7 @@ curl_multi_assign curl_multi_cleanup curl_multi_fdset curl_multi_get_handles +curl_multi_get_offt curl_multi_info_read curl_multi_init curl_multi_perform diff --git a/lib/multi.c b/lib/multi.c index 194cf75d40..d1677f5595 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -479,6 +479,7 @@ CURLMcode curl_multi_add_handle(CURLM *m, CURL *d) /* add the easy handle to the process set */ Curl_uint_bset_add(&multi->process, data->mid); ++multi->xfers_alive; + ++multi->xfers_total_ever; Curl_cpool_xfer_init(data); multi_warn_debug(multi, data); @@ -3734,6 +3735,33 @@ CURL **curl_multi_get_handles(CURLM *m) return a; } +CURL_EXTERN curl_off_t curl_multi_get_offt(CURLM *m, CURLMinfo_offt info) +{ + struct Curl_multi *multi = m; + + if(!GOOD_MULTI_HANDLE(multi)) + return -1; + + switch(info) { + case CURLMINFO_XFERS_CURRENT: { + unsigned int n = Curl_uint_tbl_count(&multi->xfers); + if(n && multi->admin) + --n; + return (curl_off_t)n; + } + case CURLMINFO_XFERS_RUNNING: + return (curl_off_t)Curl_uint_bset_count(&multi->process); + case CURLMINFO_XFERS_PENDING: + return (curl_off_t)Curl_uint_bset_count(&multi->pending); + case CURLMINFO_XFERS_DONE: + return (curl_off_t)Curl_uint_bset_count(&multi->msgsent); + case CURLMINFO_XFERS_ADDED: + return multi->xfers_total_ever; + default: + return -1; + } +} + CURLcode Curl_multi_xfer_buf_borrow(struct Curl_easy *data, char **pbuf, size_t *pbuflen) { diff --git a/lib/multihandle.h b/lib/multihandle.h index dbad164bc5..7fbbb09e48 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -95,6 +95,7 @@ struct Curl_multi { unsigned int xfers_alive; /* amount of added transfers that have not yet reached COMPLETE state */ + curl_off_t xfers_total_ever; /* total of added transfers, ever. */ struct uint_tbl xfers; /* transfers added to this multi */ /* Each transfer's mid may be present in at most one of these */ struct uint_bset process; /* transfer being processed */ diff --git a/lib/uint-bset.c b/lib/uint-bset.c index e612c390a6..1c60f22adf 100644 --- a/lib/uint-bset.c +++ b/lib/uint-bset.c @@ -79,8 +79,9 @@ UNITTEST unsigned int Curl_uint_bset_capacity(struct uint_bset *bset) { return bset->nslots * 64; } +#endif -UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset) +unsigned int Curl_uint_bset_count(struct uint_bset *bset) { unsigned int i; unsigned int n = 0; @@ -90,7 +91,6 @@ UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset) } return n; } -#endif bool Curl_uint_bset_empty(struct uint_bset *bset) { diff --git a/scripts/singleuse.pl b/scripts/singleuse.pl index 6156496019..b47c85c995 100755 --- a/scripts/singleuse.pl +++ b/scripts/singleuse.pl @@ -111,6 +111,7 @@ my %api = ( 'curl_multi_cleanup' => 'API', 'curl_multi_fdset' => 'API', 'curl_multi_get_handles' => 'API', + 'curl_multi_get_offt' => 'API', 'curl_multi_info_read' => 'API', 'curl_multi_init' => 'API', 'curl_multi_perform' => 'API', diff --git a/src/tool_operate.c b/src/tool_operate.c index 7e09528844..4f29b8f512 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -207,7 +207,6 @@ static curl_off_t VmsSpecialSize(const char *name, struct per_transfer *transfers; /* first node */ static struct per_transfer *transfersl; /* last node */ -static curl_off_t all_pers; /* add_per_transfer creates a new 'per_transfer' node in the linked list of transfers */ @@ -229,8 +228,6 @@ static CURLcode add_per_transfer(struct per_transfer **per) transfersl = p; } *per = p; - all_xfers++; /* count total number of transfers added */ - all_pers++; return CURLE_OK; } @@ -259,7 +256,6 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per) transfersl = p; free(per); - all_pers--; return n; } @@ -1422,7 +1418,8 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global, char *errorbuf; *addedp = FALSE; *morep = FALSE; - if(all_pers < (global->parallel_max*2)) { + if(curl_multi_get_offt(multi, CURLMINFO_XFERS_CURRENT) < + (curl_off_t)(global->parallel_max*2)) { bool skipped = FALSE; do { result = create_transfer(global, share, addedp, &skipped); @@ -1760,7 +1757,7 @@ static CURLcode check_finished(struct parastate *s) CURLMsg *msg; bool checkmore = FALSE; struct GlobalConfig *global = s->global; - progress_meter(global, &s->start, FALSE); + progress_meter(global, s->multi, &s->start, FALSE); do { msg = curl_multi_info_read(s->multi, &rc); if(msg) { @@ -1885,7 +1882,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global, result = check_finished(s); } - (void)progress_meter(global, &s->start, TRUE); + (void)progress_meter(global, s->multi, &s->start, TRUE); } /* Make sure to return some kind of error if there was a multi problem */ diff --git a/src/tool_progress.c b/src/tool_progress.c index 635f8e25ee..88fa1c99d1 100644 --- a/src/tool_progress.c +++ b/src/tool_progress.c @@ -134,8 +134,6 @@ static curl_off_t all_ultotal = 0; static curl_off_t all_dlalready = 0; static curl_off_t all_ulalready = 0; -curl_off_t all_xfers = 0; /* current total */ - struct speedcount { curl_off_t dl; curl_off_t ul; @@ -151,6 +149,7 @@ static struct speedcount speedstore[SPEEDCNT]; | 6 -- 9.9G 0 2 2 0:00:40 0:00:02 0:00:37 4087M */ bool progress_meter(struct GlobalConfig *global, + CURLM *multi, struct curltime *start, bool final) { @@ -184,7 +183,6 @@ bool progress_meter(struct GlobalConfig *global, curl_off_t all_ulnow = 0; bool dlknown = TRUE; bool ulknown = TRUE; - curl_off_t all_running = 0; /* in progress */ curl_off_t speed = 0; unsigned int i; stamp = now; @@ -210,8 +208,6 @@ bool progress_meter(struct GlobalConfig *global, all_ultotal += per->ultotal; per->ultotal_added = TRUE; } - if(per->added) - all_running++; } if(dlknown && all_dltotal) msnprintf(dlpercen, sizeof(dlpercen), "%3" CURL_FORMAT_CURL_OFF_T, @@ -292,8 +288,8 @@ bool progress_meter(struct GlobalConfig *global, ulpercen, /* 3 letters */ max5data(all_dlnow, buffer[0]), max5data(all_ulnow, buffer[1]), - all_xfers, - all_running, + curl_multi_get_offt(multi, CURLMINFO_XFERS_ADDED), + curl_multi_get_offt(multi, CURLMINFO_XFERS_RUNNING), time_total, time_spent, time_left, diff --git a/src/tool_progress.h b/src/tool_progress.h index 02c34d42ee..d9009504a1 100644 --- a/src/tool_progress.h +++ b/src/tool_progress.h @@ -32,10 +32,9 @@ int xferinfo_cb(void *clientp, curl_off_t ulnow); bool progress_meter(struct GlobalConfig *global, + CURLM *multi, struct curltime *start, bool final); void progress_finalize(struct per_transfer *per); -extern curl_off_t all_xfers; /* total number */ - #endif /* HEADER_CURL_TOOL_PROGRESS_H */ diff --git a/tests/data/test1135 b/tests/data/test1135 index 070d8dff4c..73668d9994 100644 --- a/tests/data/test1135 +++ b/tests/data/test1135 @@ -109,6 +109,7 @@ curl_multi_timeout curl_multi_setopt curl_multi_assign curl_multi_get_handles +curl_multi_get_offt curl_pushheader_bynum curl_pushheader_byname curl_multi_waitfds