curl: add --rate to set max request rate per time unit

--rate "12/m" - for 12 per minute or
--rate "5/h" - for 5 per hour

Removed from TODO

Closes #8671

docs/TODO

@@ -149,7 +149,6 @@
  18.4 --proxycommand
  18.5 UTF-8 filenames in Content-Disposition
  18.6 Option to make -Z merge lined based outputs on stdout
- 18.7 at least N milliseconds between requests
  18.8 Consider convenience options for JSON and XML?
  18.9 Choose the name of file in braces for complex URLs
  18.10 improve how curl works in a windows console window
@@ -999,17 +998,6 @@
 
  https://github.com/curl/curl/issues/5175
 
-18.7 at least N milliseconds between requests
-
- Allow curl command lines issue a lot of request against services that limit
- users to no more than N requests/second or similar. Could be implemented with
- an option asking that at least a certain time has elapsed since the previous
- request before the next one will be performed. Example:
-
-    $ curl "https://example.com/api?input=[1-1000]" -d yadayada --after 500
-
- See https://github.com/curl/curl/issues/3920
-
 18.8 Consider convenience options for JSON and XML?
 
  Could we add `--xml` or `--json` to add headers needed to call rest API:

docs/cmdline-opts/Makefile.inc

@@ -198,6 +198,7 @@ DPAGES = \
   quote.d \
   random-file.d \
   range.d \
+  rate.d \
   raw.d \
   referer.d \
   remote-header-name.d \

docs/cmdline-opts/rate.d

@@ -0,0 +1,33 @@
+Long: rate
+Arg: <max request rate>
+Help: Request rate for serial transfers
+Category: connection
+Example: --rate 2/s $URL
+Example: --rate 3/h $URL
+Example: --rate 14/m $URL
+Added: 7.84.0
+See-also: limit-rate retry-delay
+---
+Specify the maximum transfer frequency you allow curl to use - in number of
+transfer starts per time unit (sometimes called request rate). Without this
+option, curl will start the next transfer as fast as possible.
+
+If given several URLs and a transfer completes faster than the allowed rate,
+curl will wait until the next transfer is started to maintain the requested
+rate. This option has no effect when --parallel is used.
+
+The request rate is provided as "N/U" where N is an integer number and U is a
+time unit. Supported units are 's' (second), 'm' (minute), 'h' (hour) and 'd'
+/(day, as in a 24 hour unit). The default time unit, if no "/U" is provided,
+is number of transfers per hour.
+
+If curl is told to allow 10 requests per minute, it will not start the next
+request until 6 seconds have elapsed since the previous transfer was started.
+
+This function uses millisecond resolution. If the allowed frequency is set
+more than 1000 per second, it will instead run unrestricted.
+
+When retrying transfers, enabled with --retry, the separate retry delay logic
+is used and not this setting.
+
+If this option is used several times, the last one will be used.

docs/options-in-versions

@@ -186,6 +186,7 @@
 --quote (-Q)                         5.3
 --random-file                        7.7
 --range (-r)                         4.0
+--rate                               7.84.0
 --raw                                7.16.2
 --referer (-e)                       4.0
 --remote-header-name (-J)            7.20.0

src/tool_cfgable.h

@@ -323,6 +323,8 @@ struct GlobalConfig {
   char *libcurl;                  /* Output libcurl code to this file name */
   bool fail_early;                /* exit on first transfer error */
   bool styled_output;             /* enable fancy output style detection */
+  long ms_per_transfer;           /* start next transfer after (at least) this
+                                     many milliseconds */
 #ifdef CURLDEBUG
   bool test_event_based;
 #endif

src/tool_getparam.c

@@ -93,6 +93,7 @@ static const struct LongShort aliases[]= {
   {"*h", "trace-ascii",              ARG_FILENAME},
   {"*H", "alpn",                     ARG_BOOL},
   {"*i", "limit-rate",               ARG_STRING},
+  {"*I", "rate",                     ARG_STRING},
   {"*j", "compressed",               ARG_BOOL},
   {"*J", "tr-encoding",              ARG_BOOL},
   {"*k", "digest",                   ARG_BOOL},
@@ -738,6 +739,51 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
         config->sendpersecond = value;
       }
       break;
+      case 'I': /* --rate (request rate) */
+      {
+        /* support a few different suffixes, extract the suffix first, then
+           get the number and convert to per hour.
+           /s == per second
+           /m == per minute
+           /h == per hour (default)
+           /d == per day (24 hours)
+        */
+        char *div = strchr(nextarg, '/');
+        char number[26];
+        long denominator;
+        long numerator = 60*60*1000; /* default per hour */
+        size_t numlen = div ? (size_t)(div - nextarg) : strlen(nextarg);
+        if(numlen > sizeof(number)-1)
+          return PARAM_NUMBER_TOO_LARGE;
+        strncpy(number, nextarg, numlen);
+        number[numlen] = 0;
+        err = str2unum(&denominator, number);
+        if(err)
+          return err;
+        if(denominator < 1)
+          return PARAM_BAD_USE;
+        if(div) {
+          char unit = div[1];
+          switch(unit) {
+          case 's': /* per second */
+            numerator = 1000;
+            break;
+          case 'm': /* per minute */
+            numerator = 60*1000;
+            break;
+          case 'h': /* per hour */
+            break;
+          case 'd': /* per day */
+            numerator = 24*60*60*1000;
+            break;
+          default:
+            errorf(global, "unsupported --rate unit\n");
+            return PARAM_BAD_USE;
+          }
+        }
+        global->ms_per_transfer = numerator/denominator;
+      }
+      break;
 
       case 'j': /* --compressed */
         if(toggle &&

src/tool_listhelp.c

@@ -559,6 +559,9 @@ const struct helptxt helptext[] = {
   {"-r, --range <range>",
    "Retrieve only the bytes within RANGE",
    CURLHELP_HTTP | CURLHELP_FTP | CURLHELP_SFTP | CURLHELP_FILE},
+  {"    --rate <max request rate>",
+   "Request rate for serial transfers",
+   CURLHELP_CONNECTION},
   {"    --raw",
    "Do HTTP \"raw\"; no transfer decoding",
    CURLHELP_HTTP},

src/tool_operate.c

@@ -2359,8 +2359,9 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
   }
   for(per = transfers; per;) {
     bool retry;
-    long delay;
+    long delay_ms;
     bool bailout = FALSE;
+    struct timeval start;
     result = pre_transfer(global, per);
     if(result)
       break;
@@ -2372,6 +2373,7 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
         break;
     }
 #endif
+    start = tvnow();
 #ifdef CURLDEBUG
     if(global->test_event_based)
       result = curl_easy_perform_ev(per->curl);
@@ -2379,9 +2381,9 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
 #endif
       result = curl_easy_perform(per->curl);
 
-    returncode = post_per_transfer(global, per, result, &retry, &delay);
+    returncode = post_per_transfer(global, per, result, &retry, &delay_ms);
     if(retry) {
-      tool_go_sleep(delay);
+      tool_go_sleep(delay_ms);
       continue;
     }
 
@@ -2399,6 +2401,18 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
 
     if(bailout)
       break;
+
+    if(per && global->ms_per_transfer) {
+      /* how long time did the most recent transfer take in number of
+         milliseconds */
+      long milli = tvdiff(tvnow(), start);
+      if(milli < global->ms_per_transfer) {
+        notef(global, "Transfer took %ld ms, waits %ldms as set by --rate\n",
+              milli, global->ms_per_transfer - milli);
+        /* The transfer took less time than wanted. Wait a little. */
+        tool_go_sleep(global->ms_per_transfer - milli);
+      }
+    }
   }
   if(returncode)
     /* returncode errors have priority */