romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-08-25 12:13:37 +03:00
Code Issues Projects Releases Packages Wiki Activity

curl: speed up handling of many URLs

Browse source 
By properly keeping track of the last entry in the list of URLs/uploads
to handle, curl now avoids many meaningless traverses of the list which
speeds up many-URL handling *MASSIVELY* (several magnitudes on 100K
URLs).

Added test 1291, to verify that it doesn't take ages - but we don't have
any detection of "too slow" command in the test suite.

Reported-by: arainchik on github
Fixes #1959
Closes #2052
This commit is contained in:
Daniel Stenberg 2017-11-04 12:56:30 +01:00
parent 90abb74ff0
commit ee8016b3de
4 changed files with 63 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/tool_cfgable.h
View file

@ -114,6 +114,7 @@ struct OperationConfig {
struct getout *url_last; /* point to the last/current node */
struct getout *url_get; /* point to the node to fill in URL */
struct getout *url_out; /* point to the node to fill in outfile */
struct getout *url_ul; /* point to the node to fill in upload */
char *cipher_list;
char *proxy_cipher_list;
char *cert;

20
src/tool_getparam.c
View file

@ -787,7 +787,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
url = config->url_get;
else
/* there was no free node, create one! */
url = new_getout(config);
config->url_get = url = new_getout(config);
if(!url)
return PARAM_NO_MEM;
@ -1787,7 +1787,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
url = config->url_out;
else
/* there was no free node, create one! */
url = new_getout(config);
config->url_out = url = new_getout(config);
if(!url)
return PARAM_NO_MEM;
@ -1912,23 +1912,23 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
/* we are uploading */
{
struct getout *url;
if(!config->url_out)
config->url_out = config->url_list;
if(config->url_out) {
if(!config->url_ul)
config->url_ul = config->url_list;
if(config->url_ul) {
/* there's a node here, if it already is filled-in continue to find
an "empty" node */
while(config->url_out && (config->url_out->flags & GETOUT_UPLOAD))
config->url_out = config->url_out->next;
while(config->url_ul && (config->url_ul->flags & GETOUT_UPLOAD))
config->url_ul = config->url_ul->next;
}
/* now there might or might not be an available node to fill in! */
if(config->url_out)
if(config->url_ul)
/* existing node */
url = config->url_out;
url = config->url_ul;
else
/* there was no free node, create one! */
url = new_getout(config);
config->url_ul = url = new_getout(config);
if(!url)
return PARAM_NO_MEM;