diff --git a/src/tool_operate.c b/src/tool_operate.c index 3e6038d9a6..a8d928f496 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -185,156 +185,6 @@ static curl_off_t VmsSpecialSize(const char *name, } #endif /* __VMS */ -/* linked-list structure for the 429 delay */ -struct curl_429_list { - char *origin; - time_t startat; - struct curl_429_list *next; -}; - -/* a list of origins and their delay timer for 429 errors */ -static struct curl_429_list *http_429_list = NULL; - -/* - * Set the delay for new transfers for a given origin. - * If the origin is not found, a new item is appended to the list. - * If the list is NULL a new list will be created. - * On error the function returns NULL, otherwise the list is returned. - */ -static struct curl_429_list *curl_429_delay_set(struct curl_429_list *list, - char *origin, - uint32_t delayms) -{ - struct curl_429_list *item = NULL; - struct curl_429_list *last_item = NULL; - struct curl_429_list *new_item = NULL; - time_t startat = delayms ? time(NULL) + (delayms / 1000) : 0; - - /* try to find the origin in the existing list and update it's timer */ - if(list) { - item = list; - do { - if(curl_strequal(item->origin, origin)) { - item->startat = startat; - return list; - } - last_item = item; - item = item->next; - } while(item); - } - - /* origin not found, so we create a new item */ - new_item = curlx_malloc(sizeof(struct curl_429_list)); - if(!new_item) - return NULL; - new_item->origin = origin; - if(!new_item->origin) { - curlx_free(new_item); - return NULL; - } - new_item->startat = startat; - new_item->next = NULL; - if(!list) - return new_item; /* the new item is the newly created list */ - last_item->next = new_item; - return list; -} - -/* - * Check the list if it contains the given origin and return it. - * Returns NULL if the origin was not found in the list. - */ -static struct curl_429_list *curl_429_delay_get(struct curl_429_list *list, - const char *origin) -{ - struct curl_429_list *item = list; - - /* loop through the list to find this origin */ - while(item) { - if(curl_strequal(item->origin, origin)) { - return item; - } - item = item->next; - } - - return NULL; /* origin not found */ -} - -/* Free all the elements and their data. */ -static void curl_429_delay_free_all(struct curl_429_list *list) -{ - struct curl_429_list *next; - struct curl_429_list *item; - - if(!list) - return; - - item = list; - do { - next = item->next; - curlx_free(item); - item = next; - } while(next); -} - -/* - * extract the host, port and scheme from the URL and write it to porigin - * porigin needs to be freed by the user of this function - * if the URL is invalid origin is set to "-/-/-" - */ -static CURLcode set_per_transfer_origin(const char *url, char **porigin) -{ - char *host = NULL; - char *port = NULL; - char *scheme = NULL; - char *origin = NULL; - size_t len_origin = 0; - CURLcode err = CURLE_OK; - CURLUcode uerr = CURLUE_OK; - CURLU *uh = curl_url(); - - uerr = curl_url_set(uh, CURLUPART_URL, url, CURLU_GUESS_SCHEME); - if(uerr) - goto urlerr; - uerr = curl_url_get(uh, CURLUPART_HOST, &host, CURLU_URLDECODE); - if(uerr) - goto urlerr; - uerr = curl_url_get(uh, CURLUPART_PORT, &port, CURLU_DEFAULT_PORT); - if(uerr) - goto urlerr; - uerr = curl_url_get(uh, CURLUPART_SCHEME, &scheme, CURLU_DEFAULT_SCHEME); - if(uerr) - goto urlerr; - - len_origin = strlen(host) + strlen(port) + strlen(scheme) + 3; - origin = curlx_malloc(len_origin); - if(!origin) { - err = CURLE_OUT_OF_MEMORY; - goto clean; - } - curl_msnprintf(origin, len_origin, "%s/%s/%s", host, port, scheme); - *porigin = origin; - err = CURLE_OK; - goto clean; - -urlerr: - origin = curlx_strdup("-/-/-"); - if(!origin) { - err = CURLE_OUT_OF_MEMORY; - goto clean; - } - *porigin = origin; - err = CURLE_OK; - goto clean; - -clean: - curl_free(host); - curl_free(port); - curl_free(scheme); - curl_url_cleanup(uh); - return err; -} - struct per_transfer *transfers; /* first node */ static struct per_transfer *transfersl; /* last node */ @@ -388,7 +238,6 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per) curlx_free(per->uploadfile); curlx_free(per->outfile); curlx_free(per->url); - curlx_free(per->origin); curl_easy_cleanup(per->curl); curlx_free(per); @@ -588,7 +437,6 @@ static CURLcode retrycheck(struct OperationConfig *config, bool *retryp, uint32_t *delayms) { - long http_error = 0; CURL *curl = per->curl; struct OutStruct *outs = &per->outs; enum retryreason reason = RETRY_NO; @@ -617,7 +465,6 @@ static CURLcode retrycheck(struct OperationConfig *config, /* This was HTTP(S) */ long response = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - http_error = response; switch(response) { case 408: /* Request Timeout */ @@ -676,17 +523,6 @@ static CURLcode retrycheck(struct OperationConfig *config, /* no retry */ return CURLE_OK; per->retry_remaining--; - per->num_retries++; - *delayms = sleeptime; - - /* The retry delay for 429 applies to all requests to the same host. - Update the global 429 delay list for this host. */ - if(http_error == 429) { - http_429_list = curl_429_delay_set(http_429_list, - per->origin, *delayms); - if(!http_429_list) - return CURLE_OUT_OF_MEMORY; - } /* Skip truncation of outfile if auto-resume is enabled for download and the partially received data is good. Only for HTTP GET requests in @@ -759,6 +595,8 @@ static CURLcode retrycheck(struct OperationConfig *config, outs->bytes = 0; /* clear for next round */ } } + per->num_retries++; + *delayms = sleeptime; result = CURLE_OK; } return result; @@ -856,7 +694,6 @@ static CURLcode post_close_output(struct per_transfer *per, /* Close the outs file */ if(outs->fopened && outs->stream) { rc = curlx_fclose(outs->stream); - outs->stream = NULL; if(!result && rc) { /* something went wrong in the writing process */ result = CURLE_WRITE_ERROR; @@ -1593,11 +1430,6 @@ static CURLcode create_single(struct OperationConfig *config, if(!per->url) break; - /* store the origin in the per transfer struct for 429 delay checks */ - result = set_per_transfer_origin(per->url, &per->origin); - if(result) - return result; - result = setup_outfile(config, per, u, outs, skipped); if(result) return result; @@ -1738,16 +1570,6 @@ static CURLcode add_parallel_transfers(CURLM *multi, CURLSH *share, sleeping = TRUE; continue; } - if(http_429_list) { - struct curl_429_list *item; - item = curl_429_delay_get(http_429_list, per->origin); - if(item && (time(NULL) < item->startat)) { - per->startat = item->startat; - per->added = FALSE; - sleeping = TRUE; - continue; - } - } per->added = TRUE; result = pre_transfer(per); @@ -2532,9 +2354,6 @@ static CURLcode run_all_transfers(CURLSH *share, global->noprogress = orig_noprogress; global->isatty = orig_isatty; - curl_429_delay_free_all(http_429_list); - http_429_list = NULL; - return result; } diff --git a/src/tool_operate.h b/src/tool_operate.h index bfebb87572..69b304dfc0 100644 --- a/src/tool_operate.h +++ b/src/tool_operate.h @@ -44,7 +44,6 @@ struct per_transfer { struct curltime start; /* start of this transfer */ struct curltime retrystart; char *url; - char *origin; curl_off_t urlnum; /* the index of the given URL */ char *outfile; int infd; diff --git a/tests/data/test142 b/tests/data/test142 index 790d7a9446..a7b85b1b5f 100644 --- a/tests/data/test142 +++ b/tests/data/test142 @@ -188,7 +188,7 @@ RETR %TESTNUMBER QUIT -Allocations: 190 +Allocations: 180 Maximum allocated: 150000 diff --git a/tests/data/test440 b/tests/data/test440 index a1517fe699..3ed08f4730 100644 --- a/tests/data/test440 +++ b/tests/data/test440 @@ -75,7 +75,7 @@ https://this.hsts.example./%TESTNUMBER 7 -Allocations: 170 +Allocations: 160 diff --git a/tests/data/test445 b/tests/data/test445 index 927d45c141..a652cde588 100644 --- a/tests/data/test445 +++ b/tests/data/test445 @@ -55,7 +55,7 @@ Refuse tunneling protocols through HTTP proxy 7 -Allocations: 1560 +Allocations: 1500 diff --git a/tests/data/test767 b/tests/data/test767 index 201bcec4e1..a0d96f1b6d 100644 --- a/tests/data/test767 +++ b/tests/data/test767 @@ -49,7 +49,7 @@ Accept: */* -Allocations: 140 +Allocations: 135 Maximum allocated: 136000