romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-08-04 22:26:13 +03:00
Code Issues Projects Releases Packages Wiki Activity

tool_operate: prevent over-queuing in parallel mode

Browse source 
When doing a huge amount of parallel transfers, we must not add them to
the per_transfer list frivolously since they all use memory after all.
This was previous done without really considering millions or billions
of transfers. Massive parallelism would use a lot of memory for no good
purpose.

The queue is now limited to twice the paralleism number.

This makes the 'Qd' value in the parallel progress meter mostly useless
for users, but works for now for us as a debug display.

Reported-by: justchen1369 on github
Fixes #8933
Closes #9389
This commit is contained in:
Daniel Stenberg 2022-08-31 15:57:46 +02:00
parent 7cd400a4d2
commit 838d894504
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
1 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/tool_operate.c
View file

@ -205,6 +205,7 @@ static curl_off_t VmsSpecialSize(const char *name,
struct per_transfer *transfers; /* first node */
static struct per_transfer *transfersl; /* last node */
static long all_pers;
/* add_per_transfer creates a new 'per_transfer' node in the linked
list of transfers */
@ -227,6 +228,8 @@ static CURLcode add_per_transfer(struct per_transfer **per)
}
*per = p;
all_xfers++; /* count total number of transfers added */
all_pers++;
return CURLE_OK;
}
@ -254,6 +257,7 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per)
transfersl = p;
free(per);
all_pers--;
return n;
}
@ -2194,9 +2198,11 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global,
bool sleeping = FALSE;
*addedp = FALSE;
*morep = FALSE;
result = create_transfer(global, share, addedp);
if(result)
return result;
if(all_pers < (global->parallel_max*2)) {
result = create_transfer(global, share, addedp);
if(result)
return result;
}
for(per = transfers; per && (all_added < global->parallel_max);
per = per->next) {
bool getadded = FALSE;