runtests: generate certs dynamically, bump to EC-256, tidy up

Before this patch the curl repository and source tarball distribution
contained test certificates as binary blobs. Used by runtests.

Drop these certificates in favor of generating them dynamically as
part of the build process. Both via autotools and CMake.

As part of this, improve certificates, the generator script and process,
file layout, and fix any issue to make it work fast and smooth both in
CI and local builds.

Note, cert generator scripts require OpenSSL >=1.0.2
(or LibreSSL >=3.1.0). Generation requires POSIX shell, also with CMake.
Without a POSIX shell tests relying on TLS (and stunnel) will fail.

Details:

- build: generate certs as part of the test run process.
- build, tests: generate certs in the build directory.
- binarycheck: drop concept of known binary files with hashes.
- binarycheck: move binary check logic into spacecheck and drop this
  separate checker tool.
- build: fix to clean all cert files.
- autotools: fix to not run leaf cert generators in parallel. To avoid
  confusion when updating the revocation database and counter.
- scripts: drop `scripts` subdir, merge two scripts into one,
  auto-generate root cert, allow generating multiple leafs at once.
- scripts: switch to EC-256 keys (was: RSA-2048). For key size and perf.
- scripts: drop `-x` echo, text dumps, most other output. To avoid log
  noise and make it quicker in CI.
- scripts: make it non-RSA-specific.
- scripts: delete unused code.
- scripts: use POSIX shell shebang. Some envs don't have bash (Alpine).
- scripts: pass test pseudo-secrets via the command-line. To avoid:
  ```
  + openssl genrsa -out test-ca.key -passout fd:0 2048
  Invalid password argument, starting with "fd:"
  ```
- cmake: fix to launch generator scripts via the detected POSIX shell.
- cmake: fix `build-certs` rule to not depend on `SRPFILES`
  (`srp-verifier-*`).
- cmake: drop `EXCLUDE_FROM_ALL` for the cert subdir. It makes
  the Visual Studio generator miss to create the `clean-certs`,
  `build-certs` targets. No target depend on them, so they don't execute
  implicitly anyway. Fixes:
  ```
  MSBUILD : error MSB1009: Project file does not exist.
  Switch: clean-certs.vcxproj
  ```
- cmake: add `VERBATIM USES_TERMINAL` to `build-certs` target.
- GHA/linux: install openssl on Alpine, for the cert generator scripts.

Follow-up to 556f722fe3 #16593
Follow-up to fa461b4eff #14486

Closes #16824

tests/data/test2033

@@ -40,7 +40,7 @@ simple HTTPS GET with DER public key pinning (Schannel variant)
 CURL_SSL_BACKEND=schannel
  </setenv>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey %SRCDIR/certs/test-localhost.pub.der --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey %CERTDIR/certs/test-localhost.pub.der --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2034

@@ -36,7 +36,7 @@ https test-localhost.pem
 simple HTTPS GET with DER public key pinning
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey %SRCDIR/certs/test-localhost.pub.der https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey %CERTDIR/certs/test-localhost.pub.der https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2035

@@ -27,7 +27,7 @@ https test-localhost.pem
 HTTPS wrong DER pinnedpubkey but right CN
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey %SRCDIR/certs/test-localhost.der https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey %CERTDIR/certs/test-localhost.der https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2037

@@ -36,7 +36,7 @@ https test-localhost.pem
 simple HTTPS GET with PEM public key pinning
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey %SRCDIR/certs/test-localhost.pub.pem https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey %CERTDIR/certs/test-localhost.pub.pem https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2038

@@ -27,7 +27,7 @@ https test-localhost.pem
 HTTPS wrong PEM pinnedpubkey but right CN
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey %SRCDIR/certs/test-localhost.pem https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey %CERTDIR/certs/test-localhost.pem https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2041

@@ -36,7 +36,7 @@ https test-localhost.pem
 simple HTTPS GET with base64-sha256 public key pinning
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey sha256//%sha256b64file[%SRCDIR/certs/test-localhost.pub.der]sha256b64file% https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey sha256//%sha256b64file[%CERTDIR/certs/test-localhost.pub.der]sha256b64file% https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2042

@@ -27,7 +27,7 @@ https test-localhost.pem
 HTTPS wrong base64-sha256 pinnedpubkey but right CN
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey sha256//bSIggTf+ikMG0CtmDlpMVBd7yi7H1md4URogRPqerso= https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey sha256//bSIggTf+ikMG0CtmDlpMVBd7yi7H1md4URogRPqerso= https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2048

@@ -26,7 +26,7 @@ https test-localhost.pem
 pinnedpubkey no-match must fail even when insecure
 </name>
 <command>
--4 --insecure --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey %SRCDIR/certs/test-localhost.nn.pub.der https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --insecure --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey %CERTDIR/certs/test-localhost.nn.pub.der https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2070

@@ -39,7 +39,7 @@ Ignore certificate revocation "best effort" strategy
 CURL_SSL_BACKEND=schannel
  </setenv>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2079

@@ -40,7 +40,7 @@ simple HTTPS GET with PEM public key pinning (Schannel variant)
 CURL_SSL_BACKEND=schannel
  </setenv>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey %SRCDIR/certs/test-localhost.pub.pem --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey %CERTDIR/certs/test-localhost.pub.pem --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2087

@@ -40,7 +40,7 @@ simple HTTPS GET with base64-sha256 public key pinning (Schannel variant)
 CURL_SSL_BACKEND=schannel
  </setenv>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --pinnedpubkey sha256//%sha256b64file[%SRCDIR/certs/test-localhost.pub.der]sha256b64file% --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --pinnedpubkey sha256//%sha256b64file[%CERTDIR/certs/test-localhost.pub.der]sha256b64file% --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2500

@@ -39,7 +39,7 @@ http/3
 HTTP/3 GET:
 </name>
 <command>
---cacert %SRCDIR/certs/test-ca.crt --http3 --resolve localhost:%HTTP3PORT:%HOSTIP https://localhost:%HTTP3PORT/%TESTNUMBER
+--cacert %CERTDIR/certs/test-ca.crt --http3 --resolve localhost:%HTTP3PORT:%HOSTIP https://localhost:%HTTP3PORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test2502

@@ -59,7 +59,7 @@ lib%TESTNUMBER
 HTTP GET multiple over HTTP/3
 </name>
 <command>
-https://%HOSTIP:%HTTP3PORT/path/%TESTNUMBER %HOSTIP %HTTP3PORT %SRCDIR/certs/test-ca.cacert
+https://%HOSTIP:%HTTP3PORT/path/%TESTNUMBER %HOSTIP %HTTP3PORT %CERTDIR/certs/test-ca.cacert
 </command>
 </client>
 

tests/data/test2503

@@ -41,7 +41,7 @@ HTTP/3 header-api
 <setenv>
 </setenv>
 <command>
--w '%{header_json}\n' --http3-only --cacert %SRCDIR/certs/test-ca.crt --resolve localhost:%HTTP3PORT:%HOSTIP https://localhost:%HTTP3PORT/%TESTNUMBER
+-w '%{header_json}\n' --http3-only --cacert %CERTDIR/certs/test-ca.crt --resolve localhost:%HTTP3PORT:%HOSTIP https://localhost:%HTTP3PORT/%TESTNUMBER
 </command>
 
 </client>

tests/data/test3000

@@ -35,7 +35,7 @@ https test-localhost-san-first.pem
 HTTPS GET to localhost, first subject alt name matches, CN does not match
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test3001

@@ -35,7 +35,7 @@ https test-localhost-san-last.pem
 HTTPS GET to localhost, last subject alt name matches, CN does not match
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test3023

@@ -39,7 +39,7 @@ HTTPS GET to localhost, first subject alt name matches, CN does not match (Schan
 CURL_SSL_BACKEND=schannel
 </setenv>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test3024

@@ -39,7 +39,7 @@ HTTPS GET to localhost, last subject alt name matches, CN does not match (Schann
 CURL_SSL_BACKEND=schannel
 </setenv>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test310

@@ -35,7 +35,7 @@ https test-localhost.pem
 simple HTTPS GET
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test311

@@ -26,7 +26,7 @@ https test-localhost0h.pem
 HTTPS wrong subjectAltName but right CN
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test312

@@ -26,7 +26,7 @@ https test-localhost.nn.pem
 HTTPS GET to localhost and null-prefixed CN cert
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test313

@@ -22,7 +22,7 @@ https test-localhost.pem
 CRL test
 </name>
 <command>
--4 --cacert %SRCDIR/certs/test-ca.crt --crlfile %SRCDIR/certs/test-localhost.crl https://localhost:%HTTPSPORT/%TESTNUMBER
+-4 --cacert %CERTDIR/certs/test-ca.crt --crlfile %CERTDIR/certs/test-localhost.crl https://localhost:%HTTPSPORT/%TESTNUMBER
 </command>
 </client>
 

tests/data/test3207

@@ -165,7 +165,7 @@ lib%TESTNUMBER
 </tool>
 # provide URL and ca-cert
 <command>
-https://localhost:%HTTPSPORT/%TESTNUMBER  %SRCDIR/certs/test-ca.crt
+https://localhost:%HTTPSPORT/%TESTNUMBER %CERTDIR/certs/test-ca.crt
 </command>
 </client>
 

tests/data/test417

@@ -50,7 +50,7 @@ s/^(.*):(.*)[\r\n]*//
 </stripfile>
 <stdout>
 nomnom
-%strippemfile[%SRCDIR/certs/test-localhost.crt]strippemfile%
+%strippemfile[%CERTDIR/certs/test-localhost.crt]strippemfile%
 </stdout>
 
 </verify>

tests/data/test678

@@ -37,7 +37,7 @@ lib%TESTNUMBER
 </tool>
 # provide URL and ca-cert
 <command>
-https://localhost:%HTTPSPORT/%TESTNUMBER %SRCDIR/certs/test-ca.crt
+https://localhost:%HTTPSPORT/%TESTNUMBER %CERTDIR/certs/test-ca.crt
 </command>
 # Ensure that we're running on localhost because we're checking the host name
 <precheck>