diff --git a/src/tool_operate.c b/src/tool_operate.c index 3725c6f1a6..8904859d13 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -186,82 +186,77 @@ static curl_off_t VmsSpecialSize(const char *name, /* linked-list structure for the 429 delay */ struct curl_429_list { - char *hostname; + char *origin; time_t startat; struct curl_429_list *next; }; -/* a list of hosts and their delay timer for 429 errors */ +/* a list of origins and their delay timer for 429 errors */ static struct curl_429_list *http_429_list = NULL; /* - * Set the delay for new transfers for a given host. - * If the host is not found, a new item is appended to the list. + * Set the delay for new transfers for a given origin. + * If the origin is not found, a new item is appended to the list. * If the list is NULL a new list will be created. * On error the function returns NULL, otherwise the list is returned. */ static struct curl_429_list *curl_429_delay_set(struct curl_429_list *list, - const char *host, + const char *origin, uint32_t delayms) { - struct curl_429_list *item; - struct curl_429_list *last_item; - struct curl_429_list *new_item; - time_t startat = time(NULL) + (delayms / 1000); + struct curl_429_list *item = NULL; + struct curl_429_list *last_item = NULL; + struct curl_429_list *new_item = NULL; + time_t startat = delayms ? time(NULL) + (delayms / 1000) : 0; - /* if there is no list yet we create a new list */ - if(!list) { - new_item = curlx_malloc(sizeof(struct curl_429_list)); - if(!new_item) - return NULL; - - new_item->hostname = curlx_strdup(host); - new_item->startat = startat; - new_item->next = NULL; - return new_item; + /* try to find the origin in the existing list and update it's timer */ + if(list) { + item = list; + do { + if(curl_strequal(item->origin, origin)) { + item->startat = startat; + return list; + } + last_item = item; + item = item->next; + } while(item); } - /* loop through the list to check if we already got this host */ - item = list; - do { - if(curl_strequal(item->hostname, host)) { - item->startat = startat; - return list; - } - last_item = item; - item = item->next; - } while(item); - - /* host was not found so we create a new item and append it to the list */ + /* origin not found, so we create a new item */ new_item = curlx_malloc(sizeof(struct curl_429_list)); if(!new_item) return NULL; - - new_item->hostname = curlx_strdup(host); + new_item->origin = curlx_strdup(origin); + if(!new_item->origin) { + curlx_free(new_item); + return NULL; + } new_item->startat = startat; new_item->next = NULL; + if(!list) + return new_item; /* the new item is the newly created list */ last_item->next = new_item; return list; } /* - * Check the list if it contains the given host and return it. - * Returns NULL if the host was not found in the list. + * Check the list if it contains the given origin and return it. + * Returns NULL if the origin was not found in the list. */ static struct curl_429_list *curl_429_delay_get(struct curl_429_list *list, - const char *host) + const char *origin) { struct curl_429_list *item = list; - /* loop through the list to find this host */ + /* loop through the list to find this origin */ while(item) { - if(curl_strequal(item->hostname, host)) { + if(curl_strequal(item->origin, origin)) { return item; } item = item->next; } - return NULL; /* host not found */ + return NULL; /* origin not found */ } /* Free all the elements and their data. */ @@ -276,12 +271,70 @@ static void curl_429_delay_free_all(struct curl_429_list *list) item = list; do { next = item->next; - curlx_safefree(item->hostname); + curlx_safefree(item->origin); curlx_free(item); item = next; } while(next); } +/* + * extract the host, port and scheme from the URL and write it to porigin + * porigin needs to be freed by the user of this function + * if the URL is invalid origin is set to "-/-/-" + */ +static CURLcode set_per_transfer_origin(const char *url, char **porigin) +{ + char *host = NULL; + char *port = NULL; + char *scheme = NULL; + char *origin = NULL; + size_t len_origin = 0; + CURLcode err = CURLE_OK; + CURLUcode uerr = CURLUE_OK; + CURLU *uh = curl_url(); + + uerr = curl_url_set(uh, CURLUPART_URL, url, CURLU_GUESS_SCHEME); + if(uerr) + goto urlerr; + uerr = curl_url_get(uh, CURLUPART_HOST, &host, CURLU_URLDECODE); + if(uerr) + goto urlerr; + uerr = curl_url_get(uh, CURLUPART_PORT, &port, CURLU_DEFAULT_PORT); + if(uerr) + goto urlerr; + uerr = curl_url_get(uh, CURLUPART_SCHEME, &scheme, CURLU_DEFAULT_SCHEME); + if(uerr) + goto urlerr; + + len_origin = strlen(host) + strlen(port) + strlen(scheme) + 3; + origin = curlx_malloc(len_origin); + if(!origin) { + err = CURLE_OUT_OF_MEMORY; + goto clean; + } + curl_msnprintf(origin, len_origin, "%s/%s/%s", host, port, scheme); + *porigin = origin; + err = CURLE_OK; + goto clean; + +urlerr: + origin = curlx_strdup("-/-/-"); + if(!origin) { + err = CURLE_OUT_OF_MEMORY; + goto clean; + } + *porigin = origin; + err = CURLE_OK; + goto clean; + +clean: + curl_free(host); + curl_free(port); + curl_free(scheme); + curl_url_cleanup(uh); + return err; +} + struct per_transfer *transfers; /* first node */ static struct per_transfer *transfersl; /* last node */ @@ -335,6 +388,7 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per) curlx_free(per->uploadfile); curlx_free(per->outfile); curlx_free(per->url); + curlx_free(per->origin); curl_easy_cleanup(per->curl); curlx_free(per); @@ -534,7 +588,7 @@ static CURLcode retrycheck(struct OperationConfig *config, bool *retryp, uint32_t *delayms) { - bool is_http_429_error = FALSE; + long http_error = 0; CURL *curl = per->curl; struct OutStruct *outs = &per->outs; enum retryreason reason = RETRY_NO; @@ -563,11 +617,11 @@ static CURLcode retrycheck(struct OperationConfig *config, /* This was HTTP(S) */ long response = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); + http_error = response; switch(response) { case 408: /* Request Timeout */ case 429: /* Too Many Requests (RFC6585) */ - is_http_429_error = TRUE; case 500: /* Internal Server Error */ case 502: /* Bad Gateway */ case 503: /* Service Unavailable */ @@ -627,15 +681,11 @@ static CURLcode retrycheck(struct OperationConfig *config, /* The retry delay for 429 applies to all requests to the same host. Update the global 429 delay list for this host. */ - if(is_http_429_error) { - char *host = NULL; - CURLU *hurl = curl_url(); - curl_url_set(hurl, CURLUPART_URL, per->url, CURLU_GUESS_SCHEME); - curl_url_get(hurl, CURLUPART_HOST, &host, CURLU_URLDECODE); - http_429_list = curl_429_delay_set(http_429_list, host, *delayms); + if(http_error == 429) { + http_429_list = curl_429_delay_set(http_429_list, + per->origin, *delayms); if(!http_429_list) return CURLE_OUT_OF_MEMORY; - curl_url_cleanup(hurl); } /* Skip truncation of outfile if auto-resume is enabled for download and @@ -1543,6 +1593,11 @@ static CURLcode create_single(struct OperationConfig *config, if(!per->url) break; + /* store the origin in the per transfer struct for 429 delay checks */ + result = set_per_transfer_origin(per->url, &per->origin); + if(result) + return result; + result = setup_outfile(config, per, u, outs, skipped); if(result) return result; @@ -1684,12 +1739,8 @@ static CURLcode add_parallel_transfers(CURLM *multi, CURLSH *share, continue; } if(http_429_list) { - char *host = NULL; - CURLU *hurl = curl_url(); - curl_url_set(hurl, CURLUPART_URL, per->url, CURLU_GUESS_SCHEME); - curl_url_get(hurl, CURLUPART_HOST, &host, CURLU_URLDECODE); - struct curl_429_list *item = curl_429_delay_get(http_429_list, host); - curl_url_cleanup(hurl); + struct curl_429_list *item; + item = curl_429_delay_get(http_429_list, per->origin); if(item && (time(NULL) < item->startat)) { per->startat = item->startat; per->added = FALSE; @@ -2120,8 +2171,6 @@ static CURLcode parallel_transfers(CURLSH *share) &s->more_transfers, &s->added_transfers); if(result) { curl_multi_cleanup(s->multi); - curl_429_delay_free_all(http_429_list); - http_429_list = NULL; return result; } @@ -2182,8 +2231,6 @@ static CURLcode parallel_transfers(CURLSH *share) } curl_multi_cleanup(s->multi); - curl_429_delay_free_all(http_429_list); - http_429_list = NULL; return result; } @@ -2484,6 +2531,9 @@ static CURLcode run_all_transfers(CURLSH *share, global->noprogress = orig_noprogress; global->isatty = orig_isatty; + curl_429_delay_free_all(http_429_list); + http_429_list = NULL; + return result; } diff --git a/src/tool_operate.h b/src/tool_operate.h index 69b304dfc0..bfebb87572 100644 --- a/src/tool_operate.h +++ b/src/tool_operate.h @@ -44,6 +44,7 @@ struct per_transfer { struct curltime start; /* start of this transfer */ struct curltime retrystart; char *url; + char *origin; curl_off_t urlnum; /* the index of the given URL */ char *outfile; int infd;