gnutls: fix more nettle 4+ compatibility issues

- disable DES with nettle 4. It no longer supports it.
  ```
  lib/curl_ntlm_core.c:67:12: fatal error: 'nettle/des.h' file not found
     67 | #  include <nettle/des.h>
        |            ^~~~~~~~~~~~~~
  ```

- fix MD4 support with nettle 4.
  ```
  lib/md4.c:178:36: error: too many arguments to function call, expected 2, have 3
    178 |   md4_digest(ctx, MD4_DIGEST_SIZE, digest);
        |   ~~~~~~~~~~                       ^~~~~~
  ```

- fix unused argument compiler warning:
  ```
  lib/vtls/gtls.c:2267:39: error: unused parameter 'sha256len' [clang-diagnostic-unused-parameter,-warnings-as-errors]
  2267 |                                size_t sha256len)
       |                                       ^
  ```
  Ref: https://github.com/curl/curl/actions/runs/25710321195/job/75488970143?pr=21557

- GHA/macos: stop enabling NTLM in the GnuTLS job.
  It no longer builds due to missing DES support in nettle 4.
  ```
  lib/curl_ntlm_core.c:90:4: error: "cannot compile NTLM support without a crypto library with DES."
     90 | #  error "cannot compile NTLM support without a crypto library with DES."
        |    ^
  ```
  Ref: https://github.com/curl/curl/actions/runs/25710321195/job/75488970170?pr=21557

Follow-up to cfadbaa133 #21169

Closes #21557

.github/workflows/macos.yml

@@ -354,7 +354,7 @@ jobs:
             generate: >-
               -DENABLE_DEBUG=ON -DCURL_USE_GNUTLS=ON -DCURL_USE_OPENSSL=OFF
               -DCURL_USE_GSSAPI=ON -DGSS_ROOT_DIR=/opt/homebrew/opt/krb5
-              -DCURL_DISABLE_LDAP=ON -DUSE_SSLS_EXPORT=ON -DCURL_ENABLE_NTLM=ON
+              -DCURL_DISABLE_LDAP=ON -DUSE_SSLS_EXPORT=ON
 
           - name: 'aws-lc +analyzer'
             compiler: gcc-15

lib/curl_ntlm_core.c

@@ -49,6 +49,13 @@
      in NTLM type-3 messages.
  */
 
+#ifdef USE_GNUTLS
+#include <nettle/version.h>
+#if NETTLE_VERSION_MAJOR < 4
+#define HAVE_GNUTLS_DES
+#endif
+#endif
+
 #if defined(USE_OPENSSL) && defined(HAVE_DES_ECB_ENCRYPT)
 
 #  include <openssl/des.h>
@@ -63,7 +70,7 @@
 #  include <wolfssl/wolfcrypt/des3.h>
 #  define USE_WOLFSSL_DES
 
-#elif defined(USE_GNUTLS)
+#elif defined(HAVE_GNUTLS_DES)
 #  include <nettle/des.h>
 #  define USE_CURL_DES_SET_ODD_PARITY
 #elif defined(USE_MBEDTLS) && defined(HAVE_MBEDTLS_DES_CRYPT_ECB)

lib/md4.c

@@ -158,6 +158,7 @@ static void my_md4_final(unsigned char *digest, my_md4_ctx *ctx)
 
 #elif defined(USE_GNUTLS)
 #include <nettle/md4.h>
+#include <nettle/version.h>
 
 typedef struct md4_ctx my_md4_ctx;
 
@@ -175,7 +176,11 @@ static void my_md4_update(my_md4_ctx *ctx,
 
 static void my_md4_final(unsigned char *digest, my_md4_ctx *ctx)
 {
+#if NETTLE_VERSION_MAJOR >= 4
+  md4_digest(ctx, digest);
+#else
   md4_digest(ctx, MD4_DIGEST_SIZE, digest);
+#endif
 }
 
 #else

lib/vtls/gtls.c

@@ -2270,6 +2270,7 @@ static CURLcode gtls_sha256sum(const unsigned char *tmp, /* input */
   sha256_init(&SHA256pw);
   sha256_update(&SHA256pw, (unsigned int)tmplen, tmp);
 #if NETTLE_VERSION_MAJOR >= 4
+  (void)sha256len;
   sha256_digest(&SHA256pw, sha256sum);
 #else
   sha256_digest(&SHA256pw, (unsigned int)sha256len, sha256sum);