From 40f2da6ec3d032b1645c17cfa8dffeaaf7052c2a Mon Sep 17 00:00:00 2001
From: Viktor Szakats <commit@vsz.me>
Date: Wed, 27 May 2026 10:06:34 +0200
Subject: [PATCH] vtls: more large buffer support and error checks for SHA-256

- gnutls: support 4GiB+ SHA-256 digest inputs.
- openssl: check success of low-level update/finish digest calls.
- openssl: pass NULL to `EVP_DigestFinal_ex()` instead of discarding
  returned value.
- wolfssl: support 4GiB+ SHA-256 digest inputs.
- wolfssl: check success of low-level update/finish digest calls.
- sync and tidy up argument names in low-level sha256_sum functions.

Closes #21771
---
 lib/vtls/gtls.c    | 11 ++++++++---
 lib/vtls/openssl.c | 18 ++++++++++--------
 lib/vtls/wolfssl.c | 15 +++++++++++----
 3 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/lib/vtls/gtls.c b/lib/vtls/gtls.c
index 37cf96e08c..dcda203bb7 100644
--- a/lib/vtls/gtls.c
+++ b/lib/vtls/gtls.c
@@ -2269,14 +2269,19 @@ static CURLcode gtls_random(struct Curl_easy *data,
   return rc ? CURLE_FAILED_INIT : CURLE_OK;
 }
 
-static CURLcode gtls_sha256sum(const unsigned char *tmp, /* input */
-                               size_t tmplen,
+static CURLcode gtls_sha256sum(const unsigned char *input,
+                               size_t len,
                                unsigned char *sha256sum, /* output */
                                size_t sha256len)
 {
   struct sha256_ctx SHA256pw;
   sha256_init(&SHA256pw);
-  sha256_update(&SHA256pw, (unsigned int)tmplen, tmp);
+  do {
+    unsigned int ilen = (unsigned int)CURLMIN(len, UINT_MAX);
+    sha256_update(&SHA256pw, ilen, input);
+    len -= ilen;
+    input += ilen;
+  } while(len);
 #if NETTLE_VERSION_MAJOR >= 4
   (void)sha256len;
   sha256_digest(&SHA256pw, sha256sum);
diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index fde151590b..8789dedc79 100644
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -5454,26 +5454,28 @@ static CURLcode ossl_random(struct Curl_easy *data,
   return rc == 1 ? CURLE_OK : CURLE_FAILED_INIT;
 }
 
-static CURLcode ossl_sha256sum(const unsigned char *tmp, /* input */
-                               size_t tmplen,
+static CURLcode ossl_sha256sum(const unsigned char *input,
+                               size_t len,
                                unsigned char *sha256sum /* output */,
                                size_t unused)
 {
+  CURLcode result = CURLE_OK;
   EVP_MD_CTX *mdctx;
-  unsigned int len = 0;
   (void)unused;
 
   mdctx = EVP_MD_CTX_create();
   if(!mdctx)
     return CURLE_OUT_OF_MEMORY;
   if(!EVP_DigestInit(mdctx, EVP_sha256())) {
-    EVP_MD_CTX_destroy(mdctx);
-    return CURLE_FAILED_INIT;
+    result = CURLE_FAILED_INIT;
+    goto out;
   }
-  EVP_DigestUpdate(mdctx, tmp, tmplen);
-  EVP_DigestFinal_ex(mdctx, sha256sum, &len);
+  if(!EVP_DigestUpdate(mdctx, input, len) ||
+     !EVP_DigestFinal_ex(mdctx, sha256sum, NULL))
+    result = CURLE_BAD_FUNCTION_ARGUMENT;
+out:
   EVP_MD_CTX_destroy(mdctx);
-  return CURLE_OK;
+  return result;
 }
 
 static bool ossl_cert_status_request(void)
diff --git a/lib/vtls/wolfssl.c b/lib/vtls/wolfssl.c
index 26d260ae0f..96ad6554f4 100644
--- a/lib/vtls/wolfssl.c
+++ b/lib/vtls/wolfssl.c
@@ -2284,8 +2284,8 @@ static CURLcode wssl_random(struct Curl_easy *data,
   return CURLE_OK;
 }
 
-static CURLcode wssl_sha256sum(const unsigned char *tmp, /* input */
-                               size_t tmplen,
+static CURLcode wssl_sha256sum(const unsigned char *input,
+                               size_t len,
                                unsigned char *sha256sum /* output */,
                                size_t unused)
 {
@@ -2293,8 +2293,15 @@ static CURLcode wssl_sha256sum(const unsigned char *tmp, /* input */
   (void)unused;
   if(wc_InitSha256(&SHA256pw))
     return CURLE_FAILED_INIT;
-  wc_Sha256Update(&SHA256pw, tmp, (word32)tmplen);
-  wc_Sha256Final(&SHA256pw, sha256sum);
+  do {
+    word32 ilen = (word32)CURLMIN(len, UINT32_MAX);
+    if(wc_Sha256Update(&SHA256pw, input, ilen))
+      return CURLE_BAD_FUNCTION_ARGUMENT;
+    len -= ilen;
+    input += ilen;
+  } while(len);
+  if(wc_Sha256Final(&SHA256pw, sha256sum))
+    return CURLE_BAD_FUNCTION_ARGUMENT;
   return CURLE_OK;
 }