romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-07-31 21:18:04 +03:00
Code Issues Projects Releases Packages Wiki Activity

HTTP/3: add proxy CONNECT and MASQUE CONNECT-UDP support (ngtcp2 QUIC)

Browse source 
This patch adds two major proxy capabilities to curl (ngtcp2 QUIC):
- HTTP/3 Proxy CONNECT: Tunnel HTTP/1.1 or HTTP/2 traffic through an
  HTTPS proxy that speaks HTTP/3 (QUIC) using the standard CONNECT
  method over an HTTP/3 connection.
- MASQUE CONNECT-UDP: Tunnel HTTP/3 (QUIC) traffic through an HTTP
  proxy (speaking HTTP/1.1, HTTP/2, or HTTP/3) using the extended
  CONNECT method with the CONNECT-UDP protocol (RFC9297 & RFC9298).

Public API additions:
- `CURLPROXY_HTTPS3`: new proxy type constant for HTTP/3 proxy
- `--proxy-http3`: new CLI flag to negotiate HTTP/3 with HTTPS proxy

The implementation adds two new filters:
- `H3-PROXY` - enables negotiating HTTP/3 (QUIC) to the proxy and
  running CONNECT/CONNECT-UDP through that proxy transport.
- `CAPSULE` - dedicated filter inserted between QUIC transport and
  HTTP-PROXY to handle datagram capsule encapsulation/decapsulation.

Here is how the curl filter chaining looks in different scenarios:
- HTTP/3 Proxy CONNECT (tunneling TCP protocols over QUIC proxy):
  conn -> HTTP/1.1 or HTTP/2  -> SSL -> HTTP-PROXY ->
                                 H3-PROXY -> HAPPY-EYEBALLS -> UDP
- MASQUE CONNECT-UDP (tunneling QUIC over any proxy):
  conn -> HTTP/3 -> CAPSULE -> HTTP-PROXY -> H3-PROXY ->
                               HAPPY-EYEBALLS -> UDP
  conn -> HTTP/3 -> CAPSULE -> HTTP-PROXY -> H1-PROXY or H2-PROXY ->
                               SSL -> HAPPY-EYEBALLS -> TCP

- Both features currently require the ngtcp2 QUIC backend.
- Both features are experimental (disabled by default). Enable with
  `--enable-proxy-http3`(autotools) or `-DUSE_PROXY_HTTP3=ON`(CMake).

Tests:
- tests/unit/unit3400.c: Unit tests for capsule protocol encode/decode
- tests/http/test_60_h3_proxy.py: Comprehensive pytest integration suite
- tests/http/testenv/h2o.py: Managing h2o instances with HTTP/1.1, HTTP/2,
  and HTTP/3 (QUIC) listeners, proxy.connect and proxy.connect-udp enabled.

References:
  RFC 9297 - HTTP Datagrams and the Capsule Protocol
  RFC 9298 - Proxying UDP in HTTP
  RFC 9000 §16 — Variable-Length Integer Encoding

Signed-off-by: Aritra Basu <aritrbas+gh@cisco.com>

Closes #21153
This commit is contained in:
Aritra Basu 2026-04-27 19:35:38 -04:00 • committed by Daniel Stenberg
parent efc3f2309e
commit e78b1b3ecc
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
66 changed files with 7401 additions and 473 deletions
Download patch file Download diff file Expand all files Collapse all files

9
tests/http/testenv/curl.py
View file

@ -688,7 +688,12 @@ class CurlClient:
proxy_name = '[::1]' if use_ipv6 else \
self._server_addr if use_ip else self.env.proxy_domain
if proxys:
pport = self.env.pts_port(proto) if tunnel else self.env.proxys_port
if tunnel:
pport = self.env.pts_port(proto)
elif proto == 'h3':
pport = self.env.h3proxys_port
else:
pport = self.env.proxys_port
xargs = [
'--proxy', f'https://{proxy_name}:{pport}/',
'--proxy-cacert', self.env.ca.cert_file,
@ -697,6 +702,8 @@ class CurlClient:
xargs.extend(['--resolve', f'{proxy_name}:{pport}:{self._server_addr}'])
if proto == 'h2':
xargs.append('--proxy-http2')
elif proto == 'h3':
xargs.append('--proxy-http3')
else:
xargs = [
'--proxy', f'http://{proxy_name}:{self.env.proxy_port}/',

491
tests/http/testenv/env.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -54,20 +54,21 @@ def init_config_from(conf_path):
TESTS_HTTPD_PATH = os.path.dirname(os.path.dirname(__file__))
PROJ_PATH = os.path.dirname(os.path.dirname(TESTS_HTTPD_PATH))
TOP_PATH = os.path.join(os.getcwd(), os.path.pardir)
CONFIG_PATH = os.path.join(TOP_PATH, 'tests', 'http', 'config.ini')
CONFIG_PATH = os.path.join(TOP_PATH, "tests", "http", "config.ini")
if not os.path.exists(CONFIG_PATH):
ALT_CONFIG_PATH = os.path.join(PROJ_PATH, 'tests', 'http', 'config.ini')
ALT_CONFIG_PATH = os.path.join(PROJ_PATH, "tests", "http", "config.ini")
if not os.path.exists(ALT_CONFIG_PATH):
raise Exception(f'unable to find config.ini in {CONFIG_PATH} nor {ALT_CONFIG_PATH}')
raise Exception(
f"unable to find config.ini in {CONFIG_PATH} nor {ALT_CONFIG_PATH}"
)
TOP_PATH = PROJ_PATH
CONFIG_PATH = ALT_CONFIG_PATH
DEF_CONFIG = init_config_from(CONFIG_PATH)
CURL = os.path.join(TOP_PATH, 'src', 'curl')
CURLINFO = os.path.join(TOP_PATH, 'src', 'curlinfo')
CURL = os.path.join(TOP_PATH, "src", "curl")
CURLINFO = os.path.join(TOP_PATH, "src", "curlinfo")
class NghttpxUtil:
CMD = None
VERSION_FULL = None
@ -76,34 +77,37 @@ class NghttpxUtil:
if cmd is None:
return None
if cls.VERSION_FULL is None or cmd != cls.CMD:
p = subprocess.run(args=[cmd, '--version'],
capture_output=True, text=True)
p = subprocess.run(args=[cmd, "--version"], capture_output=True, text=True)
if p.returncode != 0:
raise RuntimeError(f'{cmd} --version failed with exit code: {p.returncode}')
raise RuntimeError(
f"{cmd} --version failed with exit code: {p.returncode}"
)
cls.CMD = cmd
for line in p.stdout.splitlines(keepends=False):
if line.startswith('nghttpx '):
if line.startswith("nghttpx "):
cls.VERSION_FULL = line
if cls.VERSION_FULL is None:
raise RuntimeError(f'{cmd}: unable to determine version')
raise RuntimeError(f"{cmd}: unable to determine version")
return cls.VERSION_FULL
@staticmethod
def version_with_h3(version):
return re.match(r'.* ngtcp2/\d+\.\d+\.\d+.*', version) is not None
return re.match(r".* ngtcp2/\d+\.\d+\.\d+.*", version) is not None
class EnvConfig:
def __init__(self, pytestconfig: Optional[pytest.Config] = None,
testrun_uid=None,
worker_id=None):
def __init__(
self,
pytestconfig: Optional[pytest.Config] = None,
testrun_uid=None,
worker_id=None,
):
self.pytestconfig = pytestconfig
self.testrun_uid = testrun_uid
self.worker_id = worker_id if worker_id is not None else 'master'
self.worker_id = worker_id if worker_id is not None else "master"
self.tests_dir = TESTS_HTTPD_PATH
self.gen_root = self.gen_dir = os.path.join(self.tests_dir, 'gen')
if self.worker_id != 'master':
self.gen_root = self.gen_dir = os.path.join(self.tests_dir, "gen")
if self.worker_id != "master":
self.gen_dir = os.path.join(self.gen_dir, self.worker_id)
self.project_dir = os.path.dirname(os.path.dirname(self.tests_dir))
self.build_dir = TOP_PATH
@ -111,57 +115,56 @@ class EnvConfig:
# check cur and its features
self.curl = CURL
self.curlinfo = CURLINFO
if 'CURL' in os.environ:
self.curl = os.environ['CURL']
if "CURL" in os.environ:
self.curl = os.environ["CURL"]
self.curl_props = {
'version_string': '',
'version': '',
'os': '',
'fullname': '',
'features_string': '',
'features': set(),
'protocols_string': '',
'protocols': set(),
'libs': set(),
'lib_versions': set(),
"version_string": "",
"version": "",
"os": "",
"fullname": "",
"features_string": "",
"features": set(),
"protocols_string": "",
"protocols": set(),
"libs": set(),
"lib_versions": set(),
}
self.curl_is_debug = False
self.curl_protos = []
p = subprocess.run(args=[self.curl, '-V'],
capture_output=True, text=True)
p = subprocess.run(args=[self.curl, "-V"], capture_output=True, text=True)
if p.returncode != 0:
raise RuntimeError(f'{self.curl} -V failed with exit code: {p.returncode}')
if p.stderr.startswith('WARNING:'):
raise RuntimeError(f"{self.curl} -V failed with exit code: {p.returncode}")
if p.stderr.startswith("WARNING:"):
self.curl_is_debug = True
for line in p.stdout.splitlines(keepends=False):
if line.startswith('curl '):
self.curl_props['version_string'] = line
m = re.match(r'^curl (?P<version>\S+) (?P<os>\S+) (?P<libs>.*)$', line)
if line.startswith("curl "):
self.curl_props["version_string"] = line
m = re.match(r"^curl (?P<version>\S+) (?P<os>\S+) (?P<libs>.*)$", line)
if m:
self.curl_props['fullname'] = m.group(0)
self.curl_props['version'] = m.group('version')
self.curl_props['os'] = m.group('os')
self.curl_props['lib_versions'] = {
lib.lower() for lib in m.group('libs').split(' ')
self.curl_props["fullname"] = m.group(0)
self.curl_props["version"] = m.group("version")
self.curl_props["os"] = m.group("os")
self.curl_props["lib_versions"] = {
lib.lower() for lib in m.group("libs").split(" ")
}
self.curl_props['libs'] = {
re.sub(r'/[a-z0-9.-]*', '', lib) for lib in self.curl_props['lib_versions']
self.curl_props["libs"] = {
re.sub(r"/[a-z0-9.-]*", "", lib)
for lib in self.curl_props["lib_versions"]
}
if line.startswith('Features: '):
self.curl_props['features_string'] = line[10:]
self.curl_props['features'] = {
feat.lower() for feat in line[10:].split(' ')
if line.startswith("Features: "):
self.curl_props["features_string"] = line[10:]
self.curl_props["features"] = {
feat.lower() for feat in line[10:].split(" ")
}
if line.startswith('Protocols: '):
self.curl_props['protocols_string'] = line[11:]
self.curl_props['protocols'] = {
prot.lower() for prot in line[11:].split(' ')
if line.startswith("Protocols: "):
self.curl_props["protocols_string"] = line[11:]
self.curl_props["protocols"] = {
prot.lower() for prot in line[11:].split(" ")
}
p = subprocess.run(args=[self.curlinfo],
capture_output=True, text=True)
p = subprocess.run(args=[self.curlinfo], capture_output=True, text=True)
if p.returncode != 0:
raise RuntimeError(f'{self.curlinfo} failed with exit code: {p.returncode}')
raise RuntimeError(f"{self.curlinfo} failed with exit code: {p.returncode}")
self.curl_is_verbose = 'verbose-strings: ON' in p.stdout
self.curl_can_cert_status = 'cert-status: ON' in p.stdout
self.curl_override_dns = 'override-dns: ON' in p.stdout
@ -169,18 +172,18 @@ class EnvConfig:
self.ports = {}
self.httpd = self.config['httpd']['httpd']
self.apxs = self.config['httpd']['apxs']
self.httpd = self.config["httpd"]["httpd"]
self.apxs = self.config["httpd"]["apxs"]
if len(self.apxs) == 0:
self.apxs = None
self._httpd_version = None
self.examples_pem = {
'key': 'xxx',
'cert': 'xxx',
"key": "xxx",
"cert": "xxx",
}
self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs')
self.tld = 'http.curl.se'
self.htdocs_dir = os.path.join(self.gen_dir, "htdocs")
self.tld = "http.curl.se"
self.domain1 = f"one.{self.tld}"
self.domain1brotli = f"brotli.one.{self.tld}"
self.domain2 = f"two.{self.tld}"
@ -188,22 +191,43 @@ class EnvConfig:
self.proxy_domain = f"proxy.{self.tld}"
self.expired_domain = f"expired.{self.tld}"
self.cert_specs = [
CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost', '127.0.0.1'], key_type='rsa2048'),
CertificateSpec(name='domain1-no-ip', domains=[self.domain1, self.domain1brotli], key_type='rsa2048'),
CertificateSpec(name='domain1-very-bad', domains=[self.domain1, 'dns:127.0.0.1'], key_type='rsa2048'),
CertificateSpec(domains=[self.domain2], key_type='rsa2048'),
CertificateSpec(domains=[self.ftp_domain], key_type='rsa2048'),
CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'),
CertificateSpec(domains=[self.expired_domain], key_type='rsa2048',
valid_from=timedelta(days=-100), valid_to=timedelta(days=-10)),
CertificateSpec(name="clientsX", sub_specs=[
CertificateSpec(name="user1", client=True),
]),
CertificateSpec(
domains=[self.domain1, self.domain1brotli, "localhost", "127.0.0.1"],
key_type="rsa2048",
),
CertificateSpec(
name="domain1-no-ip",
domains=[self.domain1, self.domain1brotli],
key_type="rsa2048",
),
CertificateSpec(
name="domain1-very-bad",
domains=[self.domain1, "dns:127.0.0.1"],
key_type="rsa2048",
),
CertificateSpec(domains=[self.domain2], key_type="rsa2048"),
CertificateSpec(domains=[self.ftp_domain], key_type="rsa2048"),
CertificateSpec(
domains=[self.proxy_domain, "127.0.0.1"], key_type="rsa2048"
),
CertificateSpec(
domains=[self.expired_domain],
key_type="rsa2048",
valid_from=timedelta(days=-100),
valid_to=timedelta(days=-10),
),
CertificateSpec(
name="clientsX",
sub_specs=[
CertificateSpec(name="user1", client=True),
],
),
]
self.openssl = 'openssl'
p = subprocess.run(args=[self.openssl, 'version'],
capture_output=True, text=True)
self.openssl = "openssl"
p = subprocess.run(
args=[self.openssl, "version"], capture_output=True, text=True
)
if p.returncode != 0:
# no openssl in path
self.openssl = None
@ -211,7 +235,7 @@ class EnvConfig:
else:
self.openssl_version = p.stdout.strip()
self.nghttpx = self.config['nghttpx']['nghttpx']
self.nghttpx = self.config["nghttpx"]["nghttpx"]
if len(self.nghttpx.strip()) == 0:
self.nghttpx = None
self._nghttpx_version = None
@ -220,30 +244,58 @@ class EnvConfig:
self._nghttpx_version = NghttpxUtil.version(self.nghttpx)
self.nghttpx_with_h3 = NghttpxUtil.version_with_h3(self._nghttpx_version)
self.caddy = self.config['caddy']['caddy']
self.caddy = self.config["caddy"]["caddy"]
self._caddy_version = None
if len(self.caddy.strip()) == 0:
self.caddy = None
self.h2o = self.config["h2o"]["h2o"]
if len(self.h2o.strip()) == 0:
self.h2o = None
self._h2o_version = None
if self.h2o is not None:
try:
p = subprocess.run(
args=[self.h2o, "--version"], capture_output=True, text=True
)
if p.returncode != 0:
# not a working h2o
self.h2o = None
else:
# h2o --version output format: "h2o version 2.3.0"
m = re.search(r"h2o version (\S+)", p.stdout)
if m:
self._h2o_version = m.group(1)
else:
self.h2o = None
except Exception:
log.exception("checking h2o version")
self.h2o = None
if self.caddy is not None:
p = subprocess.run(args=[self.caddy, 'version'],
capture_output=True, text=True)
p = subprocess.run(
args=[self.caddy, "version"], capture_output=True, text=True
)
if p.returncode != 0:
# not a working caddy
self.caddy = None
m = re.match(r'v?(\d+\.\d+\.\d+).*', p.stdout)
m = re.match(r"v?(\d+\.\d+\.\d+).*", p.stdout)
if m:
self._caddy_version = m.group(1)
else:
raise RuntimeError(f'Unable to determine caddy version from: {p.stdout}')
raise RuntimeError(
f"Unable to determine caddy version from: {p.stdout}"
)
self.vsftpd = self.config['vsftpd']['vsftpd']
if self.vsftpd == '':
self.vsftpd = self.config["vsftpd"]["vsftpd"]
if self.vsftpd == "":
self.vsftpd = None
self._vsftpd_version = None
if self.vsftpd is not None:
with tempfile.TemporaryFile('w+') as tmp:
p = subprocess.run(args=[self.vsftpd, '-v'],
capture_output=True, text=True, stdin=tmp)
with tempfile.TemporaryFile("w+") as tmp:
p = subprocess.run(
args=[self.vsftpd, "-v"], capture_output=True, text=True, stdin=tmp
)
if p.returncode != 0:
# not a working vsftpd
self.vsftpd = None
@ -256,80 +308,83 @@ class EnvConfig:
# any data there instead.
tmp.seek(0)
ver_text = tmp.read()
m = re.match(r'vsftpd: version (\d+\.\d+\.\d+)', ver_text)
m = re.match(r"vsftpd: version (\d+\.\d+\.\d+)", ver_text)
if m:
self._vsftpd_version = m.group(1)
elif len(p.stderr) == 0:
# vsftp does not use stdout or stderr for printing its version... -.-
self._vsftpd_version = 'unknown'
self._vsftpd_version = "unknown"
else:
raise Exception(f'Unable to determine VsFTPD version from: {p.stderr}')
raise Exception(f"Unable to determine VsFTPD version from: {p.stderr}")
self.danted = self.config['danted']['danted']
if self.danted == '':
self.danted = self.config["danted"]["danted"]
if self.danted == "":
self.danted = None
self._danted_version = None
if self.danted is not None:
p = subprocess.run(args=[self.danted, '-v'],
capture_output=True, text=True)
p = subprocess.run(args=[self.danted, "-v"], capture_output=True, text=True)
assert p.returncode == 0
if p.returncode != 0:
# not a working vsftpd
self.danted = None
m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stdout)
m = re.match(r"^Dante v(\d+\.\d+\.\d+).*", p.stdout)
if not m:
m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stderr)
m = re.match(r"^Dante v(\d+\.\d+\.\d+).*", p.stderr)
if m:
self._danted_version = m.group(1)
else:
self.danted = None
raise Exception(f'Unable to determine danted version from: {p.stderr}')
raise Exception(f"Unable to determine danted version from: {p.stderr}")
self.sshd = self.config['sshd']['sshd']
if self.sshd == '':
self.sshd = self.config["sshd"]["sshd"]
if self.sshd == "":
self.sshd = None
self._sshd_version = None
if self.sshd is not None:
p = subprocess.run(args=[self.sshd, '-V'],
capture_output=True, text=True)
p = subprocess.run(args=[self.sshd, "-V"], capture_output=True, text=True)
assert p.returncode == 0
if p.returncode != 0:
self.sshd = None
else:
m = re.match(r'^OpenSSH_(\d+\.\d+.*),.*', p.stderr)
assert m, f'version: {p.stderr}'
m = re.match(r"^OpenSSH_(\d+\.\d+.*),.*", p.stderr)
assert m, f"version: {p.stderr}"
if m:
self._sshd_version = m.group(1)
else:
self.sshd = None
raise Exception(f'Unable to determine sshd version from: {p.stderr}')
raise Exception(
f"Unable to determine sshd version from: {p.stderr}"
)
if self.sshd:
self.sftpd = self.config['sshd']['sftpd']
if self.sftpd == '':
self.sftpd = self.config["sshd"]["sftpd"]
if self.sftpd == "":
self.sftpd = None
else:
self.sftpd = None
self._tcpdump = shutil.which('tcpdump')
self._tcpdump = shutil.which("tcpdump")
@property
def httpd_version(self):
if self._httpd_version is None and self.apxs is not None:
try:
p = subprocess.run(args=[self.apxs, '-q', 'HTTPD_VERSION'],
capture_output=True, text=True)
p = subprocess.run(
args=[self.apxs, "-q", "HTTPD_VERSION"],
capture_output=True,
text=True,
)
if p.returncode != 0:
log.error(f'{self.apxs} failed to query HTTPD_VERSION: {p}')
log.error(f"{self.apxs} failed to query HTTPD_VERSION: {p}")
else:
self._httpd_version = p.stdout.strip()
except Exception:
log.exception(f'{self.apxs} failed to run')
log.exception(f"{self.apxs} failed to run")
return self._httpd_version
def versiontuple(self, v):
v = re.sub(r'(\d+\.\d+(\.\d+)?)(-\S+)?', r'\1', v)
return tuple(map(int, v.split('.')))
v = re.sub(r"(\d+\.\d+(\.\d+)?)(-\S+)?", r"\1", v)
return tuple(map(int, v.split(".")))
def httpd_is_at_least(self, minv):
if self.httpd_version is None:
@ -344,15 +399,17 @@ class EnvConfig:
return hv >= self.versiontuple(minv)
def is_complete(self) -> bool:
return os.path.isfile(self.httpd) and \
self.apxs is not None and \
os.path.isfile(self.apxs)
return (
os.path.isfile(self.httpd)
and self.apxs is not None
and os.path.isfile(self.apxs)
)
def get_incomplete_reason(self) -> Optional[str]:
if self.httpd is None or len(self.httpd.strip()) == 0:
return 'httpd not configured, see `--with-test-httpd=<path>`'
return "httpd not configured, see `--with-test-httpd=<path>`"
if not os.path.isfile(self.httpd):
return f'httpd ({self.httpd}) not found'
return f"httpd ({self.httpd}) not found"
if self.apxs is None:
return "command apxs not found (commonly provided in apache2-dev)"
if not os.path.isfile(self.apxs):
@ -371,18 +428,21 @@ class EnvConfig:
def vsftpd_version(self):
return self._vsftpd_version
@property
def h2o_version(self):
return self._h2o_version
@property
def tcpdmp(self) -> Optional[str]:
return self._tcpdump
def clear_locks(self):
ca_lock = os.path.join(self.gen_root, 'ca/ca.lock')
ca_lock = os.path.join(self.gen_root, "ca/ca.lock")
if os.path.exists(ca_lock):
os.remove(ca_lock)
class Env:
SERVER_TIMEOUT = 30 # seconds to wait for server to come up/reload
CONFIG = EnvConfig()
@ -407,98 +467,106 @@ class Env:
def have_h3_server() -> bool:
return Env.CONFIG.nghttpx_with_h3
@staticmethod
def have_h2o() -> bool:
return Env.CONFIG.h2o is not None
@staticmethod
def have_ssl_curl() -> bool:
return Env.curl_has_feature('ssl') or Env.curl_has_feature('multissl')
return Env.curl_has_feature("ssl") or Env.curl_has_feature("multissl")
@staticmethod
def have_h2_curl() -> bool:
return 'http2' in Env.CONFIG.curl_props['features']
return "http2" in Env.CONFIG.curl_props["features"]
@staticmethod
def have_h3_curl() -> bool:
return 'http3' in Env.CONFIG.curl_props['features']
return "http3" in Env.CONFIG.curl_props["features"]
@staticmethod
def have_compressed_curl() -> bool:
return 'brotli' in Env.CONFIG.curl_props['libs'] or \
'zlib' in Env.CONFIG.curl_props['libs'] or \
'zstd' in Env.CONFIG.curl_props['libs']
return (
"brotli" in Env.CONFIG.curl_props["libs"]
or "zlib" in Env.CONFIG.curl_props["libs"]
or "zstd" in Env.CONFIG.curl_props["libs"]
)
@staticmethod
def curl_uses_lib(libname: str) -> bool:
return libname.lower() in Env.CONFIG.curl_props['libs']
return libname.lower() in Env.CONFIG.curl_props["libs"]
@staticmethod
def curl_uses_any_libs(libs: List[str]) -> bool:
for libname in libs:
if libname.lower() in Env.CONFIG.curl_props['libs']:
if libname.lower() in Env.CONFIG.curl_props["libs"]:
return True
return False
@staticmethod
def curl_uses_ossl_quic() -> bool:
if Env.have_h3_curl():
return not Env.curl_uses_lib('ngtcp2') and Env.curl_uses_lib('nghttp3')
return not Env.curl_uses_lib("ngtcp2") and Env.curl_uses_lib("nghttp3")
return False
@staticmethod
def curl_version_string() -> str:
return Env.CONFIG.curl_props['version_string']
return Env.CONFIG.curl_props["version_string"]
@staticmethod
def curl_features_string() -> str:
return Env.CONFIG.curl_props['features_string']
return Env.CONFIG.curl_props["features_string"]
@staticmethod
def curl_has_feature(feature: str) -> bool:
return feature.lower() in Env.CONFIG.curl_props['features']
return feature.lower() in Env.CONFIG.curl_props["features"]
@staticmethod
def curl_protocols_string() -> str:
return Env.CONFIG.curl_props['protocols_string']
return Env.CONFIG.curl_props["protocols_string"]
@staticmethod
def curl_has_protocol(protocol: str) -> bool:
return protocol.lower() in Env.CONFIG.curl_props['protocols']
return protocol.lower() in Env.CONFIG.curl_props["protocols"]
@staticmethod
def curl_lib_version(libname: str) -> str:
prefix = f'{libname.lower()}/'
for lversion in Env.CONFIG.curl_props['lib_versions']:
prefix = f"{libname.lower()}/"
for lversion in Env.CONFIG.curl_props["lib_versions"]:
if lversion.startswith(prefix):
return lversion[len(prefix):]
return 'unknown'
return lversion[len(prefix) :]
return "unknown"
@staticmethod
def curl_lib_version_at_least(libname: str, min_version) -> bool:
lversion = Env.curl_lib_version(libname)
if lversion != 'unknown':
return Env.CONFIG.versiontuple(min_version) <= \
Env.CONFIG.versiontuple(lversion)
if lversion != "unknown":
return Env.CONFIG.versiontuple(min_version) <= Env.CONFIG.versiontuple(
lversion
)
return False
@staticmethod
def curl_lib_version_before(libname: str, lib_version) -> bool:
lversion = Env.curl_lib_version(libname)
if lversion != 'unknown':
if m := re.match(r'(\d+\.\d+\.\d+).*', lversion):
if lversion != "unknown":
if m := re.match(r"(\d+\.\d+\.\d+).*", lversion):
lversion = m.group(1)
return Env.CONFIG.versiontuple(lib_version) > \
Env.CONFIG.versiontuple(lversion)
return Env.CONFIG.versiontuple(lib_version) > Env.CONFIG.versiontuple(
lversion
)
return False
@staticmethod
def curl_os() -> str:
return Env.CONFIG.curl_props['os']
return Env.CONFIG.curl_props["os"]
@staticmethod
def curl_fullname() -> str:
return Env.CONFIG.curl_props['fullname']
return Env.CONFIG.curl_props["fullname"]
@staticmethod
def curl_version() -> str:
return Env.CONFIG.curl_props['version']
return Env.CONFIG.curl_props["version"]
@staticmethod
def curl_is_debug() -> bool:
@ -528,32 +596,31 @@ class Env:
@staticmethod
def curl_can_h3_early_data() -> bool:
return Env.curl_can_early_data() and \
Env.curl_uses_lib('ngtcp2')
return Env.curl_can_early_data() and Env.curl_uses_lib("ngtcp2")
@staticmethod
def http_protos() -> List[str]:
# http protocols we can test
if Env.have_h2_curl():
if Env.have_h3():
return ['http/1.1', 'h2', 'h3']
return ['http/1.1', 'h2']
return ['http/1.1']
return ["http/1.1", "h2", "h3"]
return ["http/1.1", "h2"]
return ["http/1.1"]
@staticmethod
def http_h1_h2_protos() -> List[str]:
# http 1+2 protocols we can test
if Env.have_h2_curl():
return ['http/1.1', 'h2']
return ['http/1.1']
return ["http/1.1", "h2"]
return ["http/1.1"]
@staticmethod
def http_mplx_protos() -> List[str]:
# http multiplexing protocols we can test
if Env.have_h2_curl():
if Env.have_h3():
return ['h2', 'h3']
return ['h2']
return ["h2", "h3"]
return ["h2"]
return []
@staticmethod
@ -572,6 +639,10 @@ class Env:
def caddy_version() -> str:
return Env.CONFIG.caddy_version
@staticmethod
def h2o_version() -> str:
return Env.CONFIG.h2o_version
@staticmethod
def caddy_is_at_least(minv) -> bool:
return Env.CONFIG.caddy_is_at_least(minv)
@ -611,21 +682,20 @@ class Env:
def __init__(self, pytestconfig=None, env_config=None):
if env_config:
Env.CONFIG = env_config
self._verbose = pytestconfig.option.verbose \
if pytestconfig is not None else 0
self._verbose = pytestconfig.option.verbose if pytestconfig is not None else 0
self._ca = None
self._test_timeout = 300.0 if self._verbose > 1 else 60.0 # seconds
def issue_certs(self):
if self._ca is None:
# ca_dir = os.path.join(self.CONFIG.gen_root, 'ca')
ca_dir = os.path.join(self.gen_dir, 'ca')
ca_dir = os.path.join(self.gen_dir, "ca")
os.makedirs(ca_dir, exist_ok=True)
lock_file = os.path.join(ca_dir, 'ca.lock')
lock_file = os.path.join(ca_dir, "ca.lock")
with FileLock(lock_file):
self._ca = TestCA.create_root(name=self.CONFIG.tld,
store_dir=ca_dir,
key_type="rsa2048")
self._ca = TestCA.create_root(
name=self.CONFIG.tld, store_dir=ca_dir, key_type="rsa2048"
)
self._ca.issue_certs(self.CONFIG.cert_specs)
if self.have_openssl():
self._ca.create_hashdir(self.openssl)
@ -714,19 +784,19 @@ class Env:
@property
def http_port(self) -> int:
return self.CONFIG.ports.get('http', 0)
return self.CONFIG.ports.get("http", 0)
@property
def https_port(self) -> int:
return self.CONFIG.ports['https']
return self.CONFIG.ports["https"]
@property
def https_only_tcp_port(self) -> int:
return self.CONFIG.ports['https-tcp-only']
return self.CONFIG.ports["https-tcp-only"]
@property
def nghttpx_https_port(self) -> int:
return self.CONFIG.ports['nghttpx_https']
return self.CONFIG.ports["nghttpx_https"]
@property
def h3_port(self) -> int:
@ -734,27 +804,35 @@ class Env:
@property
def proxy_port(self) -> int:
return self.CONFIG.ports['proxy']
return self.CONFIG.ports["proxy"]
@property
def proxys_port(self) -> int:
return self.CONFIG.ports['proxys']
return self.CONFIG.ports["proxys"]
@property
def ftp_port(self) -> int:
return self.CONFIG.ports['ftp']
return self.CONFIG.ports["ftp"]
@property
def ftps_port(self) -> int:
return self.CONFIG.ports['ftps']
return self.CONFIG.ports["ftps"]
@property
def h2proxys_port(self) -> int:
return self.CONFIG.ports['h2proxys']
return self.CONFIG.ports["h2proxys"]
def pts_port(self, proto: str = 'http/1.1') -> int:
@property
def h3proxys_port(self) -> int:
return self.CONFIG.ports["h3proxys"]
def pts_port(self, proto: str = "http/1.1") -> int:
# proxy tunnel port
return self.CONFIG.ports['h2proxys' if proto == 'h2' else 'proxys']
if proto == "h3":
return self.CONFIG.ports["h3proxys"]
if proto == "h2":
return self.CONFIG.ports["h2proxys"]
return self.CONFIG.ports["proxys"]
@property
def caddy(self) -> str:
@ -762,11 +840,11 @@ class Env:
@property
def caddy_https_port(self) -> int:
return self.CONFIG.ports['caddys']
return self.CONFIG.ports["caddys"]
@property
def caddy_http_port(self) -> int:
return self.CONFIG.ports['caddy']
return self.CONFIG.ports["caddy"]
@property
def danted(self) -> str:
@ -778,7 +856,7 @@ class Env:
@property
def ws_port(self) -> int:
return self.CONFIG.ports['ws']
return self.CONFIG.ports["ws"]
@property
def curl(self) -> str:
@ -802,58 +880,65 @@ class Env:
@property
def slow_network(self) -> bool:
return "CURL_DBG_SOCK_WBLOCK" in os.environ or \
"CURL_DBG_SOCK_WPARTIAL" in os.environ
return (
"CURL_DBG_SOCK_WBLOCK" in os.environ
or "CURL_DBG_SOCK_WPARTIAL" in os.environ
)
@property
def ci_run(self) -> bool:
return "CURL_CI" in os.environ
def port_for(self, alpn_proto: Optional[str] = None):
if alpn_proto is None or \
alpn_proto in ['h2', 'http/1.1', 'http/1.0', 'http/0.9']:
if alpn_proto is None or alpn_proto in [
"h2",
"http/1.1",
"http/1.0",
"http/0.9",
]:
return self.https_port
if alpn_proto in ['h3']:
if alpn_proto in ["h3"]:
return self.h3_port
return self.http_port
def authority_for(self, domain: str, alpn_proto: Optional[str] = None):
return f'{domain}:{self.port_for(alpn_proto=alpn_proto)}'
return f"{domain}:{self.port_for(alpn_proto=alpn_proto)}"
def make_data_file(self, indir: str, fname: str, fsize: int,
line_length: int = 1024) -> str:
def make_data_file(
self, indir: str, fname: str, fsize: int, line_length: int = 1024
) -> str:
if line_length < 11:
raise RuntimeError('line_length less than 11 not supported')
raise RuntimeError("line_length less than 11 not supported")
fpath = os.path.join(indir, fname)
s10 = "0123456789"
s = round((line_length / 10) + 1) * s10
s = s[0:line_length-11]
with open(fpath, 'w') as fd:
s = s[0 : line_length - 11]
with open(fpath, "w") as fd:
for i in range(int(fsize / line_length)):
fd.write(f"{i:09d}-{s}\n")
remain = int(fsize % line_length)
if remain != 0:
i = int(fsize / line_length) + 1
fd.write(f"{i:09d}-{s}"[0:remain-1] + "\n")
fd.write(f"{i:09d}-{s}"[0 : remain - 1] + "\n")
return fpath
def make_data_gzipbomb(self, indir: str, fname: str, fsize: int) -> str:
fpath = os.path.join(indir, fname)
gzpath = f'{fpath}.gz'
varpath = f'{fpath}.var'
gzpath = f"{fpath}.gz"
varpath = f"{fpath}.var"
with open(fpath, 'w') as fd:
fd.write('not what we are looking for!\n')
with open(fpath, "w") as fd:
fd.write("not what we are looking for!\n")
count = int(fsize / 1024)
zero1k = bytearray(1024)
with gzip.open(gzpath, 'wb') as fd:
with gzip.open(gzpath, "wb") as fd:
for _ in range(count):
fd.write(zero1k)
with open(varpath, 'w') as fd:
fd.write(f'URI: {fname}\n')
fd.write('\n')
fd.write(f'URI: {fname}.gz\n')
fd.write('Content-Type: text/plain\n')
fd.write('Content-Encoding: x-gzip\n')
fd.write('\n')
with open(varpath, "w") as fd:
fd.write(f"URI: {fname}\n")
fd.write("\n")
fd.write(f"URI: {fname}.gz\n")
fd.write("Content-Type: text/plain\n")
fd.write("Content-Encoding: x-gzip\n")
fd.write("\n")
return fpath

428
tests/http/testenv/h2o.py Normal file
View file

@ -0,0 +1,428 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
# SPDX-License-Identifier: curl
#
###########################################################################
#
import logging
import os
import signal
import socket
import subprocess
import time
from datetime import datetime, timedelta
from typing import Dict, Optional
from .curl import CurlClient
from .env import Env
from .ports import alloc_ports_and_do
log = logging.getLogger(__name__)
class H2o:
def __init__(self, env: Env, name: str, domain: str, cred_name: str):
self.env = env
self._name = name
self._domain = domain
self._port = 0 # defaults to h3_port
self._cred_name = cred_name
self._loaded_cred_name = None
self._process = None
self._tmp_dir = os.path.join(self.env.gen_dir, self._name)
self._run_dir = os.path.join(self._tmp_dir, "run")
self._conf_file = os.path.join(self._run_dir, "h2o.conf")
self._error_log = os.path.join(self._run_dir, "h2o.log")
self._pid_file = os.path.join(self._run_dir, "h2o.pid")
self._stderr = os.path.join(self._run_dir, "h2o.stderr")
self._cmd = env.CONFIG.h2o
# For proxy subclasses
self._h1_port = None
self._h2_port = None
@property
def port(self) -> int:
return self._port
@property
def h1_port(self) -> Optional[int]:
return getattr(self, "_h1_port", None)
@property
def h2_port(self) -> Optional[int]:
return getattr(self, "_h2_port", None)
def clear_logs(self):
self._rmf(self._error_log)
self._rmf(self._stderr)
def dump_logs(self):
lines = []
lines.append(f"stderr of {self._name}")
lines.append("-------------------------------------------")
self._dump_file(self._stderr, lines)
lines.append("")
lines.append(f"errorlog of {self._name}")
lines.append("-------------------------------------------")
self._dump_file(self._error_log, lines)
lines.append("")
return lines
def _rmf(self, path):
if os.path.isfile(path):
os.remove(path)
return
def _dump_file(self, path, lines):
if os.path.isfile(path):
with open(path) as fd:
for line in fd:
lines.append(line.rstrip())
def _mkpath(self, path):
if not os.path.exists(path):
os.makedirs(path)
return
def _log(self, level, msg):
getattr(log, level)(f"[{self._name}] {msg}")
def is_running(self):
if self._process:
self._process.poll()
return self._process.returncode is None
return False
def initial_start(self):
self._rmf(self._pid_file)
self._rmf(self._error_log)
self._mkpath(self._run_dir)
self.write_config()
def start(self, wait_live=True):
self._mkpath(self._tmp_dir)
self._mkpath(self._run_dir)
if self._process:
self.stop()
self._loaded_cred_name = self._cred_name
self.write_config()
args = [self._cmd, "-c", self._conf_file]
ngerr = open(self._stderr, "a")
self._process = subprocess.Popen(args=args, stderr=ngerr)
if self._process.returncode is not None:
return False
if wait_live:
time.sleep(1)
# fail fast if h2o rejected the config and already exited
self._process.poll()
if self._process.returncode is not None:
self._log("error",
f"h2o exited early (rc={self._process.returncode})"
f" - check {self._stderr} for details")
self._process = None
return False
return not wait_live or self.wait_for_state(
live=True, timeout=timedelta(seconds=Env.SERVER_TIMEOUT)
)
def stop(self, wait_dead=True):
self._mkpath(self._tmp_dir)
if self._process:
self._process.terminate()
try:
self._process.wait(timeout=5)
except subprocess.TimeoutExpired:
self._process.kill()
self._process.wait(timeout=2)
self._process = None
return not wait_dead or self.wait_for_state(
live=False, timeout=timedelta(seconds=5)
)
return True
def restart(self):
self.stop()
return self.start()
def reload(self, timeout: timedelta = timedelta(seconds=Env.SERVER_TIMEOUT)):
if self._process:
running = self._process
self._process = None
os.kill(running.pid, signal.SIGQUIT)
end_wait = datetime.now() + timedelta(seconds=5)
exited = False
if not self.start(wait_live=False):
self._process = running
return False
while datetime.now() < end_wait:
try:
self._log("debug", f"waiting for h2o({running.pid}) to exit.")
running.wait(1)
self._log(
"debug",
f"h2o({running.pid}) terminated -> {running.returncode}",
)
exited = True
break
except subprocess.TimeoutExpired:
self._log("warning", f"h2o({running.pid}), not shut down yet.")
os.kill(running.pid, signal.SIGQUIT)
if not exited and datetime.now() >= end_wait:
self._log("error", f"h2o({running.pid}), terminate forcefully.")
os.kill(running.pid, signal.SIGKILL)
running.terminate()
running.wait(1)
return self.wait_for_state(live=True, timeout=timeout)
return False
def wait_for_state(
self,
live: bool,
timeout: timedelta,
url: Optional[str] = None,
log_prefix: str = "h2o",
):
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
if url is None:
url = f"https://{self._domain}:{self._port}/"
while datetime.now() < try_until:
if live:
r = curl.http_get(
url=url, extra_args=["--trace", "curl.trace", "--trace-time"]
)
if r.exit_code == 0:
return True
else:
r = curl.http_get(url=url)
if r.exit_code != 0:
return True
time.sleep(0.1)
if live:
self._log("error", f"Server still not responding after {timeout}")
else:
self._log("debug", f"Server still responding after {timeout}")
return False
def write_config(self):
# To be overridden by subclasses
with open(self._conf_file, "w") as fd:
fd.write("# h2o test config\n")
class H2oServer(H2o):
"""h2o HTTP/3 server for testing."""
PORT_SPECS = {
"h2o_https": socket.SOCK_STREAM,
}
def __init__(self, env: Env):
super().__init__(
env=env, name="h2o-server", domain=env.domain1, cred_name=env.domain1
)
def initial_start(self):
super().initial_start()
def startup(ports: Dict[str, int]) -> bool:
self._port = ports["h2o_https"]
if self.start():
self.env.update_ports(ports)
return True
self.stop()
self._port = 0
return False
return alloc_ports_and_do(
H2oServer.PORT_SPECS, startup, self.env.gen_root, max_tries=3
)
def write_config(self):
creds = self.env.get_credentials(self._cred_name)
assert creds # convince pytype this is not None
doc_root = os.path.join(self.env.gen_dir, "docs")
self._mkpath(doc_root)
self._mkpath(self._run_dir)
# Create a simple test file
with open(os.path.join(doc_root, "data.json"), "w") as f:
f.write('{"message": "Hello from h2o HTTP/3 server"}\n')
with open(self._conf_file, "w") as fd:
fd.write(f"""# h2o HTTP/3 server configuration
server-name: "h2o-test-server"
num-threads: 1
listen: &ssl_listen
port: {self._port}
ssl:
certificate-file: {creds.cert_file}
key-file: {creds.pkey_file}
neverbleed: OFF
minimum-version: TLSv1.2
ocsp-update-interval: 0
listen:
<<: *ssl_listen
type: quic
hosts:
"{self._domain}":
paths:
"/":
file.dir: {doc_root}
http2-reprioritize-blocking-assets: ON
access-log: {self._run_dir}/access.log
error-log: {self._error_log}
""")
class H2oProxy(H2o):
"""h2o MASQUE proxy for testing."""
def __init__(self, env: Env):
super().__init__(
env=env,
name="h2o-proxy",
domain=env.proxy_domain,
cred_name=env.proxy_domain,
)
def initial_start(self):
super().initial_start()
def startup(ports: Dict[str, int]) -> bool:
self._port = ports["h3proxys"]
self._h2_port = ports["h2proxys"]
self._h1_port = ports["proxys"]
if self.start():
self.env.update_ports(ports)
return True
self.stop()
self._port = 0
self._h2_port = 0
self._h1_port = 0
return False
return alloc_ports_and_do(
{
"h3proxys": socket.SOCK_DGRAM,
"h2proxys": socket.SOCK_STREAM,
"proxys": socket.SOCK_STREAM,
},
startup,
self.env.gen_root,
max_tries=3,
)
def write_config(self):
creds = self.env.get_credentials(self._cred_name)
assert creds # convince pytype this is not None
self._mkpath(self._run_dir)
with open(self._conf_file, "w") as fd:
fd.write(f"""# h2o MASQUE proxy configuration
server-name: "h2o-test-proxy"
num-threads: 1
proxy.tunnel: ON
# HTTP/1.1 proxy listener
listen: &h1_listen
port: {getattr(self, "_h1_port", self._port)}
ssl:
certificate-file: {creds.cert_file}
key-file: {creds.pkey_file}
neverbleed: OFF
minimum-version: TLSv1.2
ocsp-update-interval: 0
# HTTP/2 proxy listener
listen: &h2_listen
port: {getattr(self, "_h2_port", self._port)}
ssl:
certificate-file: {creds.cert_file}
key-file: {creds.pkey_file}
neverbleed: OFF
minimum-version: TLSv1.2
ocsp-update-interval: 0
# HTTP/3 proxy listener (main port)
listen: &h3_listen
port: {self._port}
ssl:
certificate-file: {creds.cert_file}
key-file: {creds.pkey_file}
neverbleed: OFF
minimum-version: TLSv1.2
ocsp-update-interval: 0
# QUIC listener for HTTP/3
listen:
<<: *h3_listen
type: quic
hosts:
"{self._domain}":
paths:
"/":
proxy.connect: [+*]
proxy.ssl.verify-peer: OFF
"/.well-known/masque/udp":
proxy.connect-udp: [+*]
proxy.ssl.verify-peer: OFF
http2-reprioritize-blocking-assets: ON
access-log: {self._run_dir}/access.log
error-log: {self._error_log}
""")
def wait_for_state(
self,
live: bool,
timeout: timedelta,
url: Optional[str] = None,
log_prefix: str = "h2o",
):
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
if url is None:
url = f"https://{self.env.proxy_domain}:{self._port}/"
while datetime.now() < try_until:
if live:
r = curl.http_get(
url=url, extra_args=["--trace", "curl.trace", "--trace-time"]
)
if r.exit_code == 0:
return True
else:
r = curl.http_get(url=url)
if r.exit_code != 0:
return True
time.sleep(0.1)
if live:
self._log("error", f"Proxy still not responding after {timeout}")
else:
self._log("debug", f"Proxy still responding after {timeout}")
return False