diff --git a/lib/imap.c b/lib/imap.c index cb8f7e8cca..904fefafc5 100644 --- a/lib/imap.c +++ b/lib/imap.c @@ -1215,6 +1215,30 @@ static bool is_custom_fetch_listing(struct IMAP *imap) return FALSE; } +/* + * imap_stream_resp() + * + * Tells the pingpong layer that a very long untagged response line should be + * streamed to the download body rather than buffered whole. This applies to + * the SEARCH response, which lists every matching message number on a single + * line and can be arbitrarily long on a large mailbox. Custom requests (e.g. + * "-X SEARCH ..." or "-X UID SEARCH ...") run in the LIST state and have their + * untagged response written to the body the same way, so those are streamed + * too - except custom FETCH listings, whose data is delivered differently. + */ +static bool imap_stream_resp(struct Curl_easy *data, struct connectdata *conn) +{ + struct imap_conn *imapc = Curl_conn_meta_get(conn, CURL_META_IMAP_CONN); + struct IMAP *imap = Curl_meta_get(data, CURL_META_IMAP_EASY); + if(!imapc) + return FALSE; + if(imapc->state == IMAP_SEARCH) + return TRUE; + if(imapc->state == IMAP_LIST && imap && !is_custom_fetch_listing(imap)) + return TRUE; + return FALSE; +} + /* For LIST and SEARCH responses */ static CURLcode imap_state_listsearch_resp(struct Curl_easy *data, struct imap_conn *imapc, @@ -2284,6 +2308,7 @@ static CURLcode imap_setup_connection(struct Curl_easy *data, pp = &imapc->pp; PINGPONG_SETUP(pp, imap_pp_statemachine, imap_endofresp); + pp->stream_resp = imap_stream_resp; /* Set the default preferred authentication type and mechanism */ imapc->preftype = IMAP_TYPE_ANY; diff --git a/lib/pingpong.c b/lib/pingpong.c index b40d968b3f..cee7489098 100644 --- a/lib/pingpong.c +++ b/lib/pingpong.c @@ -286,7 +286,40 @@ CURLcode Curl_pp_readresp(struct Curl_easy *data, do { const char *line = curlx_dyn_ptr(&pp->recvbuf); - const char *nl = memchr(line, '\n', curlx_dyn_len(&pp->recvbuf)); + size_t buflen = curlx_dyn_len(&pp->recvbuf); + const char *nl = memchr(line, '\n', buflen); + + /* Stream a very long response line to the download body instead of + buffering it whole, when the protocol opts in (e.g. IMAP SEARCH on a + large mailbox). This keeps memory bounded no matter how long the line + is. Once started (pp->streaming) keep going until the terminating + newline arrives, at which point the line is complete. */ + if(pp->streaming || + (!nl && buflen >= PP_STREAM_FLUSH && + pp->stream_resp && pp->stream_resp(data, conn))) { + size_t chunk = nl ? (size_t)(nl - line + 1) : buflen; + if(chunk) { + result = Curl_client_write(data, CLIENTWRITE_BODY, line, chunk); + if(result) + return result; + } + if(nl) { + /* the streamed line is now complete */ + pp->streaming = FALSE; + if(buflen > chunk) + /* keep whatever follows (e.g. the tagged response line) */ + curlx_dyn_tail(&pp->recvbuf, buflen - chunk); + else + curlx_dyn_reset(&pp->recvbuf); + continue; /* re-scan any remaining buffered data */ + } + /* still no newline: everything so far is streamed, wait for more */ + pp->streaming = TRUE; + pp->overflow = 0; + curlx_dyn_reset(&pp->recvbuf); + break; + } + if(nl) { /* a newline is CRLF in pp-talk, so the CR is ignored as the line is not really terminated until the LF comes */ diff --git a/lib/pingpong.h b/lib/pingpong.h index 02f4961230..3d4225ce4c 100644 --- a/lib/pingpong.h +++ b/lib/pingpong.h @@ -64,7 +64,14 @@ struct pingpong { CURLcode (*statemachine)(struct Curl_easy *data, struct connectdata *conn); bool (*endofresp)(struct Curl_easy *data, struct connectdata *conn, const char *ptr, size_t len, int *code); + /* Optional. When a single response line grows past PP_STREAM_FLUSH bytes + without a terminating newline, this is called to ask the protocol whether + the partial line content should be streamed to the download body (as for a + large IMAP "* SEARCH ..." reply that lists all matching message numbers on + one line). Returns TRUE to stream, keeping the receive buffer bounded. */ + bool (*stream_resp)(struct Curl_easy *data, struct connectdata *conn); BIT(initialized); + BIT(streaming); /* streaming a long response line to the body */ BIT(pending_resp); /* set TRUE when a server response is pending or in progress, and is cleared once the last response is read */ @@ -74,6 +81,14 @@ struct pingpong { * has CURLOPT_SERVER_RESPONSE_TIMEOUT(_MS) set. */ #define PINGPONG_TIMEOUT_MS (60 * 1000) +/* When a response line exceeds this many bytes without a terminating newline + and the protocol opts in via pp->stream_resp(), its content is streamed to + the download body instead of being buffered as one whole line. This keeps + memory bounded for very large single-line responses (e.g. IMAP SEARCH on a + big mailbox) that would otherwise hit the receive buffer limit and fail with + CURLE_TOO_LARGE. */ +#define PP_STREAM_FLUSH (32 * 1024) + #define PINGPONG_SETUP(pp, s, e) \ do { \ (pp)->statemachine = s; \