curl/docs/examples/10-at-a-time.c

/***************************************************************************
 *                                  _   _ ____  _
 *  Project                     ___| | | |  _ \| |
 *                             / __| | | | |_) | |
 *                            | (__| |_| |  _ <| |___
 *                             \___|\___/|_| \_\_____|
 *
 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution. The terms
 * are also available at https://curl.se/docs/copyright.html.
 *
 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
 * copies of the Software, and permit persons to whom the Software is
 * furnished to do so, under the terms of the COPYING file.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 * SPDX-License-Identifier: curl
 *
 ***************************************************************************/
/* <DESC>
 * Download many files in parallel, in the same thread.
 * </DESC>
 */

#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>

static const char *urls[] = {
  "https://www.microsoft.com/",
  "https://opensource.org/",
  "https://www.google.com/",
  "https://www.yahoo.com/",
  "https://www.ibm.com/",
  "https://www.mysql.com/",
  "https://www.oracle.com/",
  "https://www.ripe.net/",
  "https://www.iana.org/",
  "https://www.amazon.com/",
  "https://www.netcraft.com/",
  "https://www.heise.de/",
  "https://www.chip.de/",
  "https://www.ca.com/",
  "https://www.cnet.com/",
  "https://www.mozilla.org/",
  "https://www.cnn.com/",
  "https://www.wikipedia.org/",
  "https://www.dell.com/",
  "https://www.hp.com/",
  "https://www.cert.org/",
  "https://www.mit.edu/",
  "https://www.nist.gov/",
  "https://www.ebay.com/",
  "https://www.playstation.com/",
  "https://www.uefa.com/",
  "https://www.ieee.org/",
  "https://www.apple.com/",
  "https://www.symantec.com/",
  "https://www.zdnet.com/",
  "https://www.fujitsu.com/global/",
  "https://www.supermicro.com/",
  "https://www.hotmail.com/",
  "https://www.ietf.org/",
  "https://www.bbc.co.uk/",
  "https://news.google.com/",
  "https://www.foxnews.com/",
  "https://www.msn.com/",
  "https://www.wired.com/",
  "https://www.sky.com/",
  "https://www.usatoday.com/",
  "https://www.cbs.com/",
  "https://www.nbc.com/",
  "https://slashdot.org/",
  "https://www.informationweek.com/",
  "https://apache.org/",
  "https://www.un.org/",
};

#define MAX_PARALLEL 10  /* number of simultaneous transfers */
#define NUM_URLS     (sizeof(urls) / sizeof(char *))

static size_t write_cb(char *data, size_t n, size_t l, void *userp)
{
  /* take care of the data here, ignored in this example */
  (void)data;
  (void)userp;
  return n * l;
}

static void add_transfer(CURLM *multi, unsigned int i, int *left)
{
  CURL *curl = curl_easy_init();
  if(curl) {
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
    curl_easy_setopt(curl, CURLOPT_URL, urls[i]);
    curl_easy_setopt(curl, CURLOPT_PRIVATE, urls[i]);
    curl_multi_add_handle(multi, curl);
  }
  (*left)++;
}

int main(void)
{
  CURLM *multi;

  CURLcode res = curl_global_init(CURL_GLOBAL_ALL);
  if(res)
    return (int)res;

  multi = curl_multi_init();
  if(multi) {
    CURLMsg *msg;
    unsigned int transfers = 0;
    int msgs_left = -1;
    int left = 0;

    /* Limit the amount of simultaneous connections curl should allow: */
    curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, (long)MAX_PARALLEL);

    for(transfers = 0; transfers < MAX_PARALLEL && transfers < NUM_URLS;
        transfers++)
      add_transfer(multi, transfers, &left);

    do {
      int still_alive = 1;
      curl_multi_perform(multi, &still_alive);

      /* !checksrc! disable EQUALSNULL 1 */
      while((msg = curl_multi_info_read(multi, &msgs_left)) != NULL) {
        if(msg->msg == CURLMSG_DONE) {
          char *url;
          CURL *curl = msg->easy_handle;
          curl_easy_getinfo(curl, CURLINFO_PRIVATE, &url);
          fprintf(stderr, "R: %d - %s <%s>\n",
                  msg->data.result, curl_easy_strerror(msg->data.result), url);
          curl_multi_remove_handle(multi, curl);
          curl_easy_cleanup(curl);
          left--;
        }
        else {
          fprintf(stderr, "E: CURLMsg (%d)\n", msg->msg);
        }
        if(transfers < NUM_URLS)
          add_transfer(multi, transfers++, &left);
      }
      if(left)
        curl_multi_wait(multi, NULL, 0, 1000, NULL);

    } while(left);

    curl_multi_cleanup(multi);
  }
  curl_global_cleanup();

  return EXIT_SUCCESS;
}