Merge eb3999bba7 into 33a967318c

lib/imap.c

@@ -1215,6 +1215,30 @@ static bool is_custom_fetch_listing(struct IMAP *imap)
   return FALSE;
 }
 
+/*
+ * imap_stream_resp()
+ *
+ * Tells the pingpong layer that a very long untagged response line should be
+ * streamed to the download body rather than buffered whole. This applies to
+ * the SEARCH response, which lists every matching message number on a single
+ * line and can be arbitrarily long on a large mailbox. Custom requests (e.g.
+ * "-X SEARCH ..." or "-X UID SEARCH ...") run in the LIST state and have their
+ * untagged response written to the body the same way, so those are streamed
+ * too - except custom FETCH listings, whose data is delivered differently.
+ */
+static bool imap_stream_resp(struct Curl_easy *data, struct connectdata *conn)
+{
+  struct imap_conn *imapc = Curl_conn_meta_get(conn, CURL_META_IMAP_CONN);
+  struct IMAP *imap = Curl_meta_get(data, CURL_META_IMAP_EASY);
+  if(!imapc)
+    return FALSE;
+  if(imapc->state == IMAP_SEARCH)
+    return TRUE;
+  if(imapc->state == IMAP_LIST && imap && !is_custom_fetch_listing(imap))
+    return TRUE;
+  return FALSE;
+}
+
 /* For LIST and SEARCH responses */
 static CURLcode imap_state_listsearch_resp(struct Curl_easy *data,
                                            struct imap_conn *imapc,
@@ -2284,6 +2308,7 @@ static CURLcode imap_setup_connection(struct Curl_easy *data,
 
   pp = &imapc->pp;
   PINGPONG_SETUP(pp, imap_pp_statemachine, imap_endofresp);
+  pp->stream_resp = imap_stream_resp;
 
   /* Set the default preferred authentication type and mechanism */
   imapc->preftype = IMAP_TYPE_ANY;

lib/pingpong.c

@@ -286,7 +286,40 @@ CURLcode Curl_pp_readresp(struct Curl_easy *data,
 
     do {
       const char *line = curlx_dyn_ptr(&pp->recvbuf);
-      const char *nl = memchr(line, '\n', curlx_dyn_len(&pp->recvbuf));
+      size_t buflen = curlx_dyn_len(&pp->recvbuf);
+      const char *nl = memchr(line, '\n', buflen);
+
+      /* Stream a very long response line to the download body instead of
+         buffering it whole, when the protocol opts in (e.g. IMAP SEARCH on a
+         large mailbox). This keeps memory bounded no matter how long the line
+         is. Once started (pp->streaming) keep going until the terminating
+         newline arrives, at which point the line is complete. */
+      if(pp->streaming ||
+         (!nl && buflen >= PP_STREAM_FLUSH &&
+          pp->stream_resp && pp->stream_resp(data, conn))) {
+        size_t chunk = nl ? (size_t)(nl - line + 1) : buflen;
+        if(chunk) {
+          result = Curl_client_write(data, CLIENTWRITE_BODY, line, chunk);
+          if(result)
+            return result;
+        }
+        if(nl) {
+          /* the streamed line is now complete */
+          pp->streaming = FALSE;
+          if(buflen > chunk)
+            /* keep whatever follows (e.g. the tagged response line) */
+            curlx_dyn_tail(&pp->recvbuf, buflen - chunk);
+          else
+            curlx_dyn_reset(&pp->recvbuf);
+          continue; /* re-scan any remaining buffered data */
+        }
+        /* still no newline: everything so far is streamed, wait for more */
+        pp->streaming = TRUE;
+        pp->overflow = 0;
+        curlx_dyn_reset(&pp->recvbuf);
+        break;
+      }
+
       if(nl) {
         /* a newline is CRLF in pp-talk, so the CR is ignored as
            the line is not really terminated until the LF comes */

lib/pingpong.h

@@ -64,7 +64,14 @@ struct pingpong {
   CURLcode (*statemachine)(struct Curl_easy *data, struct connectdata *conn);
   bool (*endofresp)(struct Curl_easy *data, struct connectdata *conn,
                     const char *ptr, size_t len, int *code);
+  /* Optional. When a single response line grows past PP_STREAM_FLUSH bytes
+     without a terminating newline, this is called to ask the protocol whether
+     the partial line content should be streamed to the download body (as for a
+     large IMAP "* SEARCH ..." reply that lists all matching message numbers on
+     one line). Returns TRUE to stream, keeping the receive buffer bounded. */
+  bool (*stream_resp)(struct Curl_easy *data, struct connectdata *conn);
   BIT(initialized);
+  BIT(streaming);     /* streaming a long response line to the body */
   BIT(pending_resp);  /* set TRUE when a server response is pending or in
                          progress, and is cleared once the last response is
                          read */
@@ -74,6 +81,14 @@ struct pingpong {
  * has CURLOPT_SERVER_RESPONSE_TIMEOUT(_MS) set. */
 #define PINGPONG_TIMEOUT_MS      (60 * 1000)
 
+/* When a response line exceeds this many bytes without a terminating newline
+   and the protocol opts in via pp->stream_resp(), its content is streamed to
+   the download body instead of being buffered as one whole line. This keeps
+   memory bounded for very large single-line responses (e.g. IMAP SEARCH on a
+   big mailbox) that would otherwise hit the receive buffer limit and fail with
+   CURLE_TOO_LARGE. */
+#define PP_STREAM_FLUSH          (32 * 1024)
+
 #define PINGPONG_SETUP(pp, s, e) \
   do {                           \
     (pp)->statemachine = s;      \