mirror of
https://github.com/curl/curl.git
synced 2026-07-29 03:03:08 +03:00
examples/complicated: fix warnings, bump deprecated callback, tidy up
Also: make them C89, add consts. Closes #15785
This commit is contained in:
parent
fa0ccd9f1f
commit
37fb50a858
15 changed files with 199 additions and 132 deletions
|
|
@ -30,14 +30,6 @@
|
|||
* </DESC>
|
||||
*/
|
||||
|
||||
/* Parameters */
|
||||
int max_con = 200;
|
||||
int max_total = 20000;
|
||||
int max_requests = 500;
|
||||
int max_link_per_page = 5;
|
||||
int follow_relative_links = 0;
|
||||
char *start_page = "https://www.reuters.com";
|
||||
|
||||
#include <libxml/HTMLparser.h>
|
||||
#include <libxml/xpath.h>
|
||||
#include <libxml/uri.h>
|
||||
|
|
@ -47,9 +39,18 @@ char *start_page = "https://www.reuters.com";
|
|||
#include <math.h>
|
||||
#include <signal.h>
|
||||
|
||||
int pending_interrupt = 0;
|
||||
void sighandler(int dummy)
|
||||
/* Parameters */
|
||||
static int max_con = 200;
|
||||
static int max_total = 20000;
|
||||
static int max_requests = 500;
|
||||
static size_t max_link_per_page = 5;
|
||||
static int follow_relative_links = 0;
|
||||
static const char *start_page = "https://www.reuters.com";
|
||||
|
||||
static int pending_interrupt = 0;
|
||||
static void sighandler(int dummy)
|
||||
{
|
||||
(void)dummy;
|
||||
pending_interrupt = 1;
|
||||
}
|
||||
|
||||
|
|
@ -59,7 +60,7 @@ typedef struct {
|
|||
size_t size;
|
||||
} memory;
|
||||
|
||||
size_t grow_buffer(void *contents, size_t sz, size_t nmemb, void *ctx)
|
||||
static size_t grow_buffer(void *contents, size_t sz, size_t nmemb, void *ctx)
|
||||
{
|
||||
size_t realsize = sz * nmemb;
|
||||
memory *mem = (memory*) ctx;
|
||||
|
|
@ -75,16 +76,17 @@ size_t grow_buffer(void *contents, size_t sz, size_t nmemb, void *ctx)
|
|||
return realsize;
|
||||
}
|
||||
|
||||
CURL *make_handle(char *url)
|
||||
static CURL *make_handle(const char *url)
|
||||
{
|
||||
CURL *handle = curl_easy_init();
|
||||
memory *mem;
|
||||
|
||||
/* Important: use HTTP2 over HTTPS */
|
||||
curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
||||
curl_easy_setopt(handle, CURLOPT_URL, url);
|
||||
|
||||
/* buffer body */
|
||||
memory *mem = malloc(sizeof(memory));
|
||||
mem = malloc(sizeof(memory));
|
||||
mem->size = 0;
|
||||
mem->buf = malloc(1);
|
||||
curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, grow_buffer);
|
||||
|
|
@ -117,37 +119,43 @@ CURL *make_handle(char *url)
|
|||
}
|
||||
|
||||
/* HREF finder implemented in libxml2 but could be any HTML parser */
|
||||
size_t follow_links(CURLM *multi_handle, memory *mem, char *url)
|
||||
static size_t follow_links(CURLM *multi_handle, memory *mem, const char *url)
|
||||
{
|
||||
int opts = HTML_PARSE_NOBLANKS | HTML_PARSE_NOERROR | \
|
||||
HTML_PARSE_NOWARNING | HTML_PARSE_NONET;
|
||||
htmlDocPtr doc = htmlReadMemory(mem->buf, mem->size, url, NULL, opts);
|
||||
htmlDocPtr doc = htmlReadMemory(mem->buf, (int)mem->size, url, NULL, opts);
|
||||
size_t count;
|
||||
int i;
|
||||
xmlChar *xpath;
|
||||
xmlNodeSetPtr nodeset;
|
||||
xmlXPathContextPtr context;
|
||||
xmlXPathObjectPtr result;
|
||||
if(!doc)
|
||||
return 0;
|
||||
xmlChar *xpath = (xmlChar*) "//a/@href";
|
||||
xmlXPathContextPtr context = xmlXPathNewContext(doc);
|
||||
xmlXPathObjectPtr result = xmlXPathEvalExpression(xpath, context);
|
||||
xpath = (xmlChar*) "//a/@href";
|
||||
context = xmlXPathNewContext(doc);
|
||||
result = xmlXPathEvalExpression(xpath, context);
|
||||
xmlXPathFreeContext(context);
|
||||
if(!result)
|
||||
return 0;
|
||||
xmlNodeSetPtr nodeset = result->nodesetval;
|
||||
nodeset = result->nodesetval;
|
||||
if(xmlXPathNodeSetIsEmpty(nodeset)) {
|
||||
xmlXPathFreeObject(result);
|
||||
return 0;
|
||||
}
|
||||
size_t count = 0;
|
||||
int i;
|
||||
count = 0;
|
||||
for(i = 0; i < nodeset->nodeNr; i++) {
|
||||
double r = rand();
|
||||
int x = r * nodeset->nodeNr / RAND_MAX;
|
||||
int x = (int)(r * nodeset->nodeNr / RAND_MAX);
|
||||
const xmlNode *node = nodeset->nodeTab[x]->xmlChildrenNode;
|
||||
xmlChar *href = xmlNodeListGetString(doc, node, 1);
|
||||
char *link;
|
||||
if(follow_relative_links) {
|
||||
xmlChar *orig = href;
|
||||
href = xmlBuildURI(href, (xmlChar *) url);
|
||||
xmlFree(orig);
|
||||
}
|
||||
char *link = (char *) href;
|
||||
link = (char *) href;
|
||||
if(!link || strlen(link) < 20)
|
||||
continue;
|
||||
if(!strncmp(link, "http://", 7) || !strncmp(link, "https://", 8)) {
|
||||
|
|
@ -161,17 +169,23 @@ size_t follow_links(CURLM *multi_handle, memory *mem, char *url)
|
|||
return count;
|
||||
}
|
||||
|
||||
int is_html(char *ctype)
|
||||
static int is_html(char *ctype)
|
||||
{
|
||||
return ctype != NULL && strlen(ctype) > 10 && strstr(ctype, "text/html");
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
CURLM *multi_handle;
|
||||
int msgs_left;
|
||||
int pending;
|
||||
int complete;
|
||||
int still_running;
|
||||
|
||||
signal(SIGINT, sighandler);
|
||||
LIBXML_TEST_VERSION;
|
||||
LIBXML_TEST_VERSION
|
||||
curl_global_init(CURL_GLOBAL_DEFAULT);
|
||||
CURLM *multi_handle = curl_multi_init();
|
||||
multi_handle = curl_multi_init();
|
||||
curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_con);
|
||||
curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 6L);
|
||||
|
||||
|
|
@ -183,17 +197,18 @@ int main(void)
|
|||
/* sets html start page */
|
||||
curl_multi_add_handle(multi_handle, make_handle(start_page));
|
||||
|
||||
int msgs_left;
|
||||
int pending = 0;
|
||||
int complete = 0;
|
||||
int still_running = 1;
|
||||
pending = 0;
|
||||
complete = 0;
|
||||
still_running = 1;
|
||||
while(still_running && !pending_interrupt) {
|
||||
int numfds;
|
||||
CURLMsg *m;
|
||||
|
||||
curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
|
||||
curl_multi_perform(multi_handle, &still_running);
|
||||
|
||||
/* See how the transfers went */
|
||||
CURLMsg *m = NULL;
|
||||
m = NULL;
|
||||
while((m = curl_multi_info_read(multi_handle, &msgs_left))) {
|
||||
if(m->msg == CURLMSG_DONE) {
|
||||
CURL *handle = m->easy_handle;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue