From eb3999bba7c8ff2baea032cb7a3e2b5d695ec75a Mon Sep 17 00:00:00 2001 From: "Allan V. C. Quadros" Date: Wed, 29 Jul 2026 13:28:09 -0500 Subject: [PATCH] imap: stream large SEARCH responses instead of buffering whole A SEARCH reply is a single untagged line listing every matching message number. On a large mailbox that line can exceed the pingpong receive buffer (DYN_PINGPPONG_CMD, 64 KB), which made dyn_nappend() abort the transfer with CURLE_TOO_LARGE and deliver nothing. Stream an over-long response line to the body as it arrives and finalize it at the CRLF, keeping memory bounded. IMAP opts in via pp->stream_resp for the SEARCH state and for custom SEARCH / UID SEARCH (LIST state), excluding custom FETCH listings. Validated against a real Dovecot server with 20000 messages; existing IMAP tests (800-899) pass. Fixes #22435 Signed-off-by: Allan V. C. Quadros --- lib/imap.c | 25 +++++++++++++++++++++++++ lib/pingpong.c | 35 ++++++++++++++++++++++++++++++++++- lib/pingpong.h | 15 +++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/lib/imap.c b/lib/imap.c index cb8f7e8cca..904fefafc5 100644 --- a/lib/imap.c +++ b/lib/imap.c @@ -1215,6 +1215,30 @@ static bool is_custom_fetch_listing(struct IMAP *imap) return FALSE; } +/* + * imap_stream_resp() + * + * Tells the pingpong layer that a very long untagged response line should be + * streamed to the download body rather than buffered whole. This applies to + * the SEARCH response, which lists every matching message number on a single + * line and can be arbitrarily long on a large mailbox. Custom requests (e.g. + * "-X SEARCH ..." or "-X UID SEARCH ...") run in the LIST state and have their + * untagged response written to the body the same way, so those are streamed + * too - except custom FETCH listings, whose data is delivered differently. + */ +static bool imap_stream_resp(struct Curl_easy *data, struct connectdata *conn) +{ + struct imap_conn *imapc = Curl_conn_meta_get(conn, CURL_META_IMAP_CONN); + struct IMAP *imap = Curl_meta_get(data, CURL_META_IMAP_EASY); + if(!imapc) + return FALSE; + if(imapc->state == IMAP_SEARCH) + return TRUE; + if(imapc->state == IMAP_LIST && imap && !is_custom_fetch_listing(imap)) + return TRUE; + return FALSE; +} + /* For LIST and SEARCH responses */ static CURLcode imap_state_listsearch_resp(struct Curl_easy *data, struct imap_conn *imapc, @@ -2284,6 +2308,7 @@ static CURLcode imap_setup_connection(struct Curl_easy *data, pp = &imapc->pp; PINGPONG_SETUP(pp, imap_pp_statemachine, imap_endofresp); + pp->stream_resp = imap_stream_resp; /* Set the default preferred authentication type and mechanism */ imapc->preftype = IMAP_TYPE_ANY; diff --git a/lib/pingpong.c b/lib/pingpong.c index b40d968b3f..cee7489098 100644 --- a/lib/pingpong.c +++ b/lib/pingpong.c @@ -286,7 +286,40 @@ CURLcode Curl_pp_readresp(struct Curl_easy *data, do { const char *line = curlx_dyn_ptr(&pp->recvbuf); - const char *nl = memchr(line, '\n', curlx_dyn_len(&pp->recvbuf)); + size_t buflen = curlx_dyn_len(&pp->recvbuf); + const char *nl = memchr(line, '\n', buflen); + + /* Stream a very long response line to the download body instead of + buffering it whole, when the protocol opts in (e.g. IMAP SEARCH on a + large mailbox). This keeps memory bounded no matter how long the line + is. Once started (pp->streaming) keep going until the terminating + newline arrives, at which point the line is complete. */ + if(pp->streaming || + (!nl && buflen >= PP_STREAM_FLUSH && + pp->stream_resp && pp->stream_resp(data, conn))) { + size_t chunk = nl ? (size_t)(nl - line + 1) : buflen; + if(chunk) { + result = Curl_client_write(data, CLIENTWRITE_BODY, line, chunk); + if(result) + return result; + } + if(nl) { + /* the streamed line is now complete */ + pp->streaming = FALSE; + if(buflen > chunk) + /* keep whatever follows (e.g. the tagged response line) */ + curlx_dyn_tail(&pp->recvbuf, buflen - chunk); + else + curlx_dyn_reset(&pp->recvbuf); + continue; /* re-scan any remaining buffered data */ + } + /* still no newline: everything so far is streamed, wait for more */ + pp->streaming = TRUE; + pp->overflow = 0; + curlx_dyn_reset(&pp->recvbuf); + break; + } + if(nl) { /* a newline is CRLF in pp-talk, so the CR is ignored as the line is not really terminated until the LF comes */ diff --git a/lib/pingpong.h b/lib/pingpong.h index 02f4961230..3d4225ce4c 100644 --- a/lib/pingpong.h +++ b/lib/pingpong.h @@ -64,7 +64,14 @@ struct pingpong { CURLcode (*statemachine)(struct Curl_easy *data, struct connectdata *conn); bool (*endofresp)(struct Curl_easy *data, struct connectdata *conn, const char *ptr, size_t len, int *code); + /* Optional. When a single response line grows past PP_STREAM_FLUSH bytes + without a terminating newline, this is called to ask the protocol whether + the partial line content should be streamed to the download body (as for a + large IMAP "* SEARCH ..." reply that lists all matching message numbers on + one line). Returns TRUE to stream, keeping the receive buffer bounded. */ + bool (*stream_resp)(struct Curl_easy *data, struct connectdata *conn); BIT(initialized); + BIT(streaming); /* streaming a long response line to the body */ BIT(pending_resp); /* set TRUE when a server response is pending or in progress, and is cleared once the last response is read */ @@ -74,6 +81,14 @@ struct pingpong { * has CURLOPT_SERVER_RESPONSE_TIMEOUT(_MS) set. */ #define PINGPONG_TIMEOUT_MS (60 * 1000) +/* When a response line exceeds this many bytes without a terminating newline + and the protocol opts in via pp->stream_resp(), its content is streamed to + the download body instead of being buffered as one whole line. This keeps + memory bounded for very large single-line responses (e.g. IMAP SEARCH on a + big mailbox) that would otherwise hit the receive buffer limit and fail with + CURLE_TOO_LARGE. */ +#define PP_STREAM_FLUSH (32 * 1024) + #define PINGPONG_SETUP(pp, s, e) \ do { \ (pp)->statemachine = s; \