diff --git a/docs/examples/crawler.c b/docs/examples/crawler.c index e2b58127d2..2ff4ae7145 100644 --- a/docs/examples/crawler.c +++ b/docs/examples/crawler.c @@ -99,7 +99,13 @@ CURL *make_handle(char *url) curl_easy_setopt(handle, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); curl_easy_setopt(handle, CURLOPT_AUTOREFERER, 1L); curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L); - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L); + /* each transfer needs to be done within 20 seconds! */ + curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 20000L); + /* connect fast or fail */ + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 2000L); + /* skip files larger than a gigabyte */ + curl_easy_setopt(handle, CURLOPT_MAXFILESIZE_LARGE, + (curl_off_t)1024*1024*1024); curl_easy_setopt(handle, CURLOPT_COOKIEFILE, ""); curl_easy_setopt(handle, CURLOPT_FILETIME, 1L); curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler");