diff --git a/docs/KNOWN_RISKS.md b/docs/KNOWN_RISKS.md index 0b22ce0b0c..2c3988bdd3 100644 --- a/docs/KNOWN_RISKS.md +++ b/docs/KNOWN_RISKS.md @@ -144,3 +144,6 @@ and secure algorithms. When asking curl or libcurl to automatically decompress data on arrival, there is a risk that the size of the output from the decompression process ends up many times larger than the input data size. + +Since curl 8.20.0, users can mitigate this risk by setting the max filesize +option that also covers the decompressed size. diff --git a/docs/cmdline-opts/max-filesize.md b/docs/cmdline-opts/max-filesize.md index e3fd900fe6..02b2293c56 100644 --- a/docs/cmdline-opts/max-filesize.md +++ b/docs/cmdline-opts/max-filesize.md @@ -37,3 +37,6 @@ threshold during transfer. Starting in curl 8.19.0, the maximum size can be specified using a fraction as in `2.5M` for two and a half megabytes. It only works with a period (`.`) delimiter, independent of what your locale might prefer. + +Since 8.20.0, this option also stops ongoing transfers that would reach this +threshold due to automatic decompression using --compressed. diff --git a/docs/libcurl/opts/CURLOPT_MAXFILESIZE.md b/docs/libcurl/opts/CURLOPT_MAXFILESIZE.md index 8618758f1b..17c4c9f4e1 100644 --- a/docs/libcurl/opts/CURLOPT_MAXFILESIZE.md +++ b/docs/libcurl/opts/CURLOPT_MAXFILESIZE.md @@ -41,6 +41,9 @@ If you want a limit above 2GB, use CURLOPT_MAXFILESIZE_LARGE(3). Since 8.4.0, this option also stops ongoing transfers if they reach this threshold. +Since 8.20.0, this option also stops ongoing transfers that would reach this +threshold due to automatic decompression using CURLOPT_ACCEPT_ENCODING(3). + # DEFAULT 0, meaning disabled. diff --git a/docs/libcurl/opts/CURLOPT_MAXFILESIZE_LARGE.md b/docs/libcurl/opts/CURLOPT_MAXFILESIZE_LARGE.md index 3ee91696e4..791b25862f 100644 --- a/docs/libcurl/opts/CURLOPT_MAXFILESIZE_LARGE.md +++ b/docs/libcurl/opts/CURLOPT_MAXFILESIZE_LARGE.md @@ -42,6 +42,9 @@ ends up being larger than this given limit. Since 8.4.0, this option also stops ongoing transfers if they reach this threshold. +Since 8.20.0, this option also stops ongoing transfers that would reach this +threshold due to automatic decompression using CURLOPT_ACCEPT_ENCODING(3). + # DEFAULT 0, meaning disabled. diff --git a/lib/cw-out.c b/lib/cw-out.c index ab38109ad9..3da803e8e9 100644 --- a/lib/cw-out.c +++ b/lib/cw-out.c @@ -31,7 +31,7 @@ #include "transfer.h" #include "cw-out.h" #include "cw-pause.h" - +#include "progress.h" /** * OVERALL DESIGN of this client writer @@ -228,8 +228,8 @@ static CURLcode cw_out_ptr_flush(struct cw_out_ctx *ctx, curl_write_callback wcb = NULL; void *wcb_data; size_t max_write, min_write; - size_t wlen, nwritten; - CURLcode result; + size_t wlen, nwritten = 0; + CURLcode result = CURLE_OK; /* If we errored once, we do not invoke the client callback again */ if(ctx->errored) @@ -253,10 +253,15 @@ static CURLcode cw_out_ptr_flush(struct cw_out_ctx *ctx, if(!flush_all && blen < min_write) break; wlen = max_write ? CURLMIN(blen, max_write) : blen; - result = cw_out_cb_write(ctx, data, wcb, wcb_data, otype, - buf, wlen, &nwritten); + if(otype == CW_OUT_BODY) + result = Curl_pgrs_deliver_check(data, wlen); + if(!result) + result = cw_out_cb_write(ctx, data, wcb, wcb_data, otype, + buf, wlen, &nwritten); if(result) return result; + if(otype == CW_OUT_BODY) + Curl_pgrs_deliver_inc(data, nwritten); *pconsumed += nwritten; blen -= nwritten; buf += nwritten; diff --git a/lib/progress.c b/lib/progress.c index 6cde5678be..2cebabc88c 100644 --- a/lib/progress.c +++ b/lib/progress.c @@ -211,6 +211,7 @@ void Curl_pgrsReset(struct Curl_easy *data) Curl_pgrsSetUploadSize(data, -1); Curl_pgrsSetDownloadSize(data, -1); data->progress.speeder_c = 0; /* reset speed records */ + data->progress.deliver = 0; pgrs_speedinit(data); } @@ -339,6 +340,26 @@ void Curl_pgrsStartNow(struct Curl_easy *data) p->ul_size_known = FALSE; } +/* check that the 'delta' amount of bytes are okay to deliver to the + application, or return error if not. */ +CURLcode Curl_pgrs_deliver_check(struct Curl_easy *data, size_t delta) +{ + if(data->set.max_filesize && + ((curl_off_t)delta > data->set.max_filesize - data->progress.deliver)) { + failf(data, "Would have exceeded max file size"); + return CURLE_FILESIZE_EXCEEDED; + } + return CURLE_OK; +} + +/* this counts how much data is delivered to the application, which + in compressed cases may differ from downloaded amount */ +void Curl_pgrs_deliver_inc(struct Curl_easy *data, size_t delta) +{ + data->progress.deliver += delta; +} + + void Curl_pgrs_download_inc(struct Curl_easy *data, size_t delta) { if(delta) { diff --git a/lib/progress.h b/lib/progress.h index 9cb0924ddc..82b0ec7e8c 100644 --- a/lib/progress.h +++ b/lib/progress.h @@ -50,7 +50,8 @@ int Curl_pgrsDone(struct Curl_easy *data); void Curl_pgrsStartNow(struct Curl_easy *data); void Curl_pgrsSetDownloadSize(struct Curl_easy *data, curl_off_t size); void Curl_pgrsSetUploadSize(struct Curl_easy *data, curl_off_t size); - +CURLcode Curl_pgrs_deliver_check(struct Curl_easy *data, size_t delta); +void Curl_pgrs_deliver_inc(struct Curl_easy *data, size_t delta); void Curl_pgrs_download_inc(struct Curl_easy *data, size_t delta); void Curl_pgrs_upload_inc(struct Curl_easy *data, size_t delta); void Curl_pgrsSetUploadCounter(struct Curl_easy *data, curl_off_t size); diff --git a/lib/urldata.h b/lib/urldata.h index b437771517..16931ef374 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -796,6 +796,7 @@ struct Progress { force redraw at next call */ struct pgrs_dir ul; struct pgrs_dir dl; + curl_off_t deliver; /* amount of data delivered to application */ curl_off_t current_speed; /* uses the currently fastest transfer */ curl_off_t earlydata_sent; diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index c442fa13c2..07b2aa7ac8 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -213,7 +213,7 @@ test1580 test1581 test1582 test1583 test1584 test1585 \ test1590 test1591 test1592 test1593 test1594 test1595 test1596 test1597 \ test1598 test1599 test1600 test1601 test1602 test1603 test1604 test1605 \ test1606 test1607 test1608 test1609 test1610 test1611 test1612 test1613 \ -test1614 test1615 test1616 test1617 \ +test1614 test1615 test1616 test1617 test1618 \ test1620 test1621 test1622 test1623 \ \ test1630 test1631 test1632 test1633 test1634 test1635 test1636 test1637 \ diff --git a/tests/data/test1618 b/tests/data/test1618 new file mode 100644 index 0000000000..4e2f46f5ad --- /dev/null +++ b/tests/data/test1618 @@ -0,0 +1,68 @@ + + + + +HTTP +HTTP GET +compressed +brotli + + + + + + +HTTP/1.1 200 OK +Date: Mon, 29 Nov 2004 21:56:53 GMT +Vary: Accept-Encoding +Content-Encoding: br +Content-Length: 14 + +%hex[%81%fa%7f%0c%fc%13%00%f1%58%20%90%7b%18%00]hex% + + + +HTTP/1.1 200 OK +Date: Mon, 29 Nov 2004 21:56:53 GMT +Vary: Accept-Encoding +Content-Encoding: br +Content-Length: 14 + + + + + +# Client-side + + +brotli + + +http + + +HTTP GET brotli compression bomb + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --compressed --max-filesize=1000 + + + +# Verify data after the test has been "shot" + + +s/^Accept-Encoding: [a-zA-Z, ]*/Accept-Encoding: xxx/ + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Accept-Encoding: xxx + + + +63 + + +