http2: support HTTP/2 to forward proxies, non-tunneling

- with `--proxy-http2` allow h2 ALPN negotiation to
  forward proxies
- applies to http: requests against a https: proxy only,
  as https: requests will auto-tunnel
- adding a HTTP/1 request parser in http1.c
- removed h2h3.c
- using new request parser in nghttp2 and all h3 backends
- adding test 2603 for request parser
- adding h2 proxy test cases to test_10_*

scorecard.py: request scoring accidentally always run curl
with '-v'. Removed that, expect double numbers.

labeller: added http1.* and h2-proxy sources to detection

Closes #10967
This commit is contained in:
Stefan Eissing 2023-04-14 11:38:14 +02:00 committed by Daniel Stenberg
parent fb1d62ff07
commit fc2f1e547a
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
28 changed files with 1522 additions and 824 deletions

View file

@ -281,7 +281,7 @@ class ScoreCard:
if max_parallel > 1 else []
self.info(f'{max_parallel}...')
for i in range(sample_size):
curl = CurlClient(env=self.env)
curl = CurlClient(env=self.env, silent=self._silent_curl)
r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True,
with_headers=False,
extra_args=extra_args)
@ -459,13 +459,11 @@ class ScoreCard:
for key, val in sval.items():
if 'errors' in val:
errors.extend(val['errors'])
print(f' {dkey:<8} {skey:>8} '
f'{self.fmt_reqs(sval["serial"]["speed"]):>12} '
f'{self.fmt_reqs(sval["par-6"]["speed"]):>12} '
f'{self.fmt_reqs(sval["par-25"]["speed"]):>12} '
f'{self.fmt_reqs(sval["par-50"]["speed"]):>12} '
f'{self.fmt_reqs(sval["par-100"]["speed"]):>12} '
f' {"/".join(errors):<20}')
line = f' {dkey:<8} {skey:>8} '
for k in sval.keys():
line += f'{self.fmt_reqs(sval[k]["speed"]):>12} '
line += f' {"/".join(errors):<20}'
print(line)
def parse_size(s):

View file

@ -50,14 +50,6 @@ class TestProxy:
httpd.clear_extra_configs()
httpd.reload()
def set_tunnel_proto(self, proto):
if proto == 'h2':
os.environ['CURL_PROXY_TUNNEL_H2'] = '1'
return 'HTTP/2'
else:
os.environ.pop('CURL_PROXY_TUNNEL_H2', None)
return 'HTTP/1.1'
def get_tunnel_proto_used(self, r: ExecResult):
for l in r.trace_lines:
m = re.match(r'.* CONNECT tunnel: (\S+) negotiated$', l)
@ -71,37 +63,60 @@ class TestProxy:
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxy', f'http://{env.proxy_domain}:{env.proxy_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxy_port}:127.0.0.1',
])
extra_args=curl.get_proxy_args(proxys=False))
r.check_response(count=1, http_status=200)
# download via https: proxy (no tunnel)
@pytest.mark.skipif(condition=not Env.curl_has_feature('HTTPS-proxy'),
reason='curl lacks HTTPS-proxy support')
@pytest.mark.parametrize("proto", ['http/1.1', 'h2'])
@pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx available")
def test_10_02_proxy_https(self, env: Env, httpd, nghttpx_fwd, repeat):
def test_10_02_proxys_down(self, env: Env, httpd, nghttpx_fwd, proto, repeat):
if proto == 'h2' and not env.curl_uses_lib('nghttp2'):
pytest.skip('only supported with nghttp2')
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(proto=proto)
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxy', f'https://{env.proxy_domain}:{env.proxys_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxys_port}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
r.check_response(count=1, http_status=200)
extra_args=xargs)
r.check_response(count=1, http_status=200,
protocol='HTTP/2' if proto == 'h2' else 'HTTP/1.1')
# upload via https: with proto (no tunnel)
@pytest.mark.skipif(condition=not Env.have_ssl_curl(), reason=f"curl without SSL")
@pytest.mark.parametrize("proto", ['http/1.1', 'h2'])
@pytest.mark.parametrize("fname, fcount", [
['data.json', 5],
['data-100k', 5],
['data-1m', 2]
])
@pytest.mark.skipif(condition=not Env.have_nghttpx(),
reason="no nghttpx available")
def test_10_02_proxys_up(self, env: Env, httpd, nghttpx, proto,
fname, fcount, repeat):
if proto == 'h2' and not env.curl_uses_lib('nghttp2'):
pytest.skip('only supported with nghttp2')
count = fcount
srcfile = os.path.join(httpd.docs_dir, fname)
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/curltest/echo?id=[0-{count-1}]'
xargs = curl.get_proxy_args(proto=proto)
r = curl.http_upload(urls=[url], data=f'@{srcfile}', alpn_proto=proto,
extra_args=xargs)
r.check_response(count=count, http_status=200,
protocol='HTTP/2' if proto == 'h2' else 'HTTP/1.1')
indata = open(srcfile).readlines()
for i in range(count):
respdata = open(curl.response_file(i)).readlines()
assert respdata == indata
# download http: via http: proxytunnel
def test_10_03_proxytunnel_http(self, env: Env, httpd, repeat):
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(proxys=False, tunnel=True)
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxytunnel',
'--proxy', f'http://{env.proxy_domain}:{env.proxy_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxy_port}:127.0.0.1',
])
extra_args=xargs)
r.check_response(count=1, http_status=200)
# download http: via https: proxytunnel
@ -111,13 +126,9 @@ class TestProxy:
def test_10_04_proxy_https(self, env: Env, httpd, nghttpx_fwd, repeat):
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(tunnel=True)
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxytunnel',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port()}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port()}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
extra_args=xargs)
r.check_response(count=1, http_status=200)
# download https: with proto via http: proxytunnel
@ -126,13 +137,10 @@ class TestProxy:
def test_10_05_proxytunnel_http(self, env: Env, httpd, proto, repeat):
curl = CurlClient(env=env)
url = f'https://localhost:{env.https_port}/data.json'
xargs = curl.get_proxy_args(proxys=False, tunnel=True)
r = curl.http_download(urls=[url], alpn_proto=proto, with_stats=True,
with_headers=True,
extra_args=[
'--proxytunnel',
'--proxy', f'http://{env.proxy_domain}:{env.proxy_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxy_port}:127.0.0.1',
])
extra_args=xargs)
r.check_response(count=1, http_status=200,
protocol='HTTP/2' if proto == 'h2' else 'HTTP/1.1')
@ -145,20 +153,15 @@ class TestProxy:
def test_10_06_proxytunnel_https(self, env: Env, httpd, nghttpx_fwd, proto, tunnel, repeat):
if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
pytest.skip('only supported with nghttp2')
exp_tunnel_proto = self.set_tunnel_proto(tunnel)
curl = CurlClient(env=env)
url = f'https://localhost:{env.https_port}/data.json?[0-0]'
xargs = curl.get_proxy_args(tunnel=True, proto=tunnel)
r = curl.http_download(urls=[url], alpn_proto=proto, with_stats=True,
with_headers=True,
extra_args=[
'--proxytunnel',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port(tunnel)}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port(tunnel)}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
with_headers=True, extra_args=xargs)
r.check_response(count=1, http_status=200,
protocol='HTTP/2' if proto == 'h2' else 'HTTP/1.1')
assert self.get_tunnel_proto_used(r) == exp_tunnel_proto
assert self.get_tunnel_proto_used(r) == 'HTTP/2' \
if tunnel == 'h2' else 'HTTP/1.1'
srcfile = os.path.join(httpd.docs_dir, 'data.json')
dfile = curl.download_file(0)
assert filecmp.cmp(srcfile, dfile, shallow=False)
@ -178,20 +181,15 @@ class TestProxy:
if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
pytest.skip('only supported with nghttp2')
count = fcount
exp_tunnel_proto = self.set_tunnel_proto(tunnel)
curl = CurlClient(env=env)
url = f'https://localhost:{env.https_port}/{fname}?[0-{count-1}]'
xargs = curl.get_proxy_args(tunnel=True, proto=tunnel)
r = curl.http_download(urls=[url], alpn_proto=proto, with_stats=True,
with_headers=True,
extra_args=[
'--proxytunnel',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port(tunnel)}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port(tunnel)}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
with_headers=True, extra_args=xargs)
r.check_response(count=count, http_status=200,
protocol='HTTP/2' if proto == 'h2' else 'HTTP/1.1')
assert self.get_tunnel_proto_used(r) == exp_tunnel_proto
assert self.get_tunnel_proto_used(r) == 'HTTP/2' \
if tunnel == 'h2' else 'HTTP/1.1'
srcfile = os.path.join(httpd.docs_dir, fname)
for i in range(count):
dfile = curl.download_file(i)
@ -213,20 +211,15 @@ class TestProxy:
pytest.skip('only supported with nghttp2')
count = fcount
srcfile = os.path.join(httpd.docs_dir, fname)
exp_tunnel_proto = self.set_tunnel_proto(tunnel)
curl = CurlClient(env=env)
url = f'https://localhost:{env.https_port}/curltest/echo?id=[0-{count-1}]'
xargs = curl.get_proxy_args(tunnel=True, proto=tunnel)
r = curl.http_upload(urls=[url], data=f'@{srcfile}', alpn_proto=proto,
extra_args=[
'--proxytunnel',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port(tunnel)}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port(tunnel)}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
assert self.get_tunnel_proto_used(r) == exp_tunnel_proto
extra_args=xargs)
assert self.get_tunnel_proto_used(r) == 'HTTP/2' \
if tunnel == 'h2' else 'HTTP/1.1'
r.check_response(count=count, http_status=200)
indata = open(srcfile).readlines()
r.check_response(count=count, http_status=200)
for i in range(count):
respdata = open(curl.response_file(i)).readlines()
assert respdata == indata
@ -237,20 +230,15 @@ class TestProxy:
def test_10_09_reuse_ser(self, env: Env, httpd, nghttpx_fwd, tunnel, repeat):
if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
pytest.skip('only supported with nghttp2')
exp_tunnel_proto = self.set_tunnel_proto(tunnel)
curl = CurlClient(env=env)
url1 = f'https://localhost:{env.https_port}/data.json'
url2 = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(tunnel=True, proto=tunnel)
r = curl.http_download(urls=[url1, url2], alpn_proto='http/1.1', with_stats=True,
with_headers=True,
extra_args=[
'--proxytunnel',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port(tunnel)}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port(tunnel)}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
with_headers=True, extra_args=xargs)
r.check_response(count=2, http_status=200)
assert self.get_tunnel_proto_used(r) == exp_tunnel_proto
assert self.get_tunnel_proto_used(r) == 'HTTP/2' \
if tunnel == 'h2' else 'HTTP/1.1'
if tunnel == 'h2':
# TODO: we would like to reuse the first connection for the
# second URL, but this is currently not possible

View file

@ -31,7 +31,7 @@ import re
import time
import pytest
from testenv import Env, CurlClient
from testenv import Env, CurlClient, ExecResult
log = logging.getLogger(__name__)
@ -52,20 +52,12 @@ class TestProxyAuth:
httpd.set_proxy_auth(False)
httpd.reload()
def set_tunnel_proto(self, proto):
if proto == 'h2':
os.environ['CURL_PROXY_TUNNEL_H2'] = '1'
return 'HTTP/2'
else:
os.environ.pop('CURL_PROXY_TUNNEL_H2', None)
return 'HTTP/1.1'
def get_tunnel_proto_used(self, curl: CurlClient):
assert os.path.exists(curl.trace_file)
for l in open(curl.trace_file).readlines():
m = re.match(r'.* == Info: CONNECT tunnel: (\S+) negotiated', l)
def get_tunnel_proto_used(self, r: ExecResult):
for line in r.trace_lines:
m = re.match(r'.* CONNECT tunnel: (\S+) negotiated$', line)
if m:
return m.group(1)
assert False, f'tunnel protocol not found in:\n{"".join(r.trace_lines)}'
return None
# download via http: proxy (no tunnel), no auth
@ -73,22 +65,17 @@ class TestProxyAuth:
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxy', f'http://{env.proxy_domain}:{env.proxy_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxy_port}:127.0.0.1',
])
extra_args=curl.get_proxy_args(proxys=False))
r.check_response(count=1, http_status=407)
# download via http: proxy (no tunnel), auth
def test_13_02_proxy_auth(self, env: Env, httpd, repeat):
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(proxys=False)
xargs.extend(['--proxy-user', 'proxy:proxy'])
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxy-user', 'proxy:proxy',
'--proxy', f'http://{env.proxy_domain}:{env.proxy_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxy_port}:127.0.0.1',
])
extra_args=xargs)
r.check_response(count=1, http_status=200)
@pytest.mark.skipif(condition=not Env.curl_has_feature('HTTPS-proxy'),
@ -97,12 +84,9 @@ class TestProxyAuth:
def test_13_03_proxys_no_auth(self, env: Env, httpd, nghttpx_fwd, repeat):
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(proxys=True)
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxy', f'https://{env.proxy_domain}:{env.pts_port()}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port()}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
extra_args=xargs)
r.check_response(count=1, http_status=407)
@pytest.mark.skipif(condition=not Env.curl_has_feature('HTTPS-proxy'),
@ -111,37 +95,28 @@ class TestProxyAuth:
def test_13_04_proxys_auth(self, env: Env, httpd, nghttpx_fwd, repeat):
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(proxys=True)
xargs.extend(['--proxy-user', 'proxy:proxy'])
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxy-user', 'proxy:proxy',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port()}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port()}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
extra_args=xargs)
r.check_response(count=1, http_status=200)
def test_13_05_tunnel_http_no_auth(self, env: Env, httpd, repeat):
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(proxys=False, tunnel=True)
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxytunnel',
'--proxy', f'http://{env.proxy_domain}:{env.proxy_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxy_port}:127.0.0.1',
])
extra_args=xargs)
# expect "COULD_NOT_CONNECT"
r.check_response(exitcode=56, http_status=None)
def test_13_06_tunnel_http_auth(self, env: Env, httpd, repeat):
curl = CurlClient(env=env)
url = f'http://localhost:{env.http_port}/data.json'
xargs = curl.get_proxy_args(proxys=False, tunnel=True)
xargs.extend(['--proxy-user', 'proxy:proxy'])
r = curl.http_download(urls=[url], alpn_proto='http/1.1', with_stats=True,
extra_args=[
'--proxytunnel',
'--proxy-user', 'proxy:proxy',
'--proxy', f'http://{env.proxy_domain}:{env.proxy_port}/',
'--resolve', f'{env.proxy_domain}:{env.proxy_port}:127.0.0.1',
])
extra_args=xargs)
r.check_response(count=1, http_status=200)
@pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx available")
@ -152,20 +127,16 @@ class TestProxyAuth:
def test_13_07_tunnels_no_auth(self, env: Env, httpd, proto, tunnel, repeat):
if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
pytest.skip('only supported with nghttp2')
exp_tunnel_proto = self.set_tunnel_proto(tunnel)
curl = CurlClient(env=env)
url = f'https://localhost:{env.https_port}/data.json'
xargs = curl.get_proxy_args(proxys=True, tunnel=True, proto=tunnel)
r = curl.http_download(urls=[url], alpn_proto=proto, with_stats=True,
with_headers=True, with_trace=True,
extra_args=[
'--proxytunnel',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port(tunnel)}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port(tunnel)}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
extra_args=xargs)
# expect "COULD_NOT_CONNECT"
r.check_response(exitcode=56, http_status=None)
assert self.get_tunnel_proto_used(curl) == exp_tunnel_proto
assert self.get_tunnel_proto_used(r) == 'HTTP/2' \
if tunnel == 'h2' else 'HTTP/1.1'
@pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx available")
@pytest.mark.skipif(condition=not Env.curl_has_feature('HTTPS-proxy'),
@ -175,19 +146,15 @@ class TestProxyAuth:
def test_13_08_tunnels_auth(self, env: Env, httpd, proto, tunnel, repeat):
if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
pytest.skip('only supported with nghttp2')
exp_tunnel_proto = self.set_tunnel_proto(tunnel)
curl = CurlClient(env=env)
url = f'https://localhost:{env.https_port}/data.json'
xargs = curl.get_proxy_args(proxys=True, tunnel=True, proto=tunnel)
xargs.extend(['--proxy-user', 'proxy:proxy'])
r = curl.http_download(urls=[url], alpn_proto=proto, with_stats=True,
with_headers=True, with_trace=True,
extra_args=[
'--proxytunnel',
'--proxy-user', 'proxy:proxy',
'--proxy', f'https://{env.proxy_domain}:{env.pts_port(tunnel)}/',
'--resolve', f'{env.proxy_domain}:{env.pts_port(tunnel)}:127.0.0.1',
'--proxy-cacert', env.ca.cert_file,
])
extra_args=xargs)
r.check_response(count=1, http_status=200,
protocol='HTTP/2' if proto == 'h2' else 'HTTP/1.1')
assert self.get_tunnel_proto_used(curl) == exp_tunnel_proto
assert self.get_tunnel_proto_used(r) == 'HTTP/2' \
if tunnel == 'h2' else 'HTTP/1.1'

View file

@ -317,6 +317,26 @@ class CurlClient:
if not os.path.exists(path):
return os.makedirs(path)
def get_proxy_args(self, proto: str = 'http/1.1',
proxys: bool = True, tunnel: bool = False):
if proxys:
pport = self.env.pts_port(proto) if tunnel else self.env.proxys_port
xargs = [
'--proxy', f'https://{self.env.proxy_domain}:{pport}/',
'--resolve', f'{self.env.proxy_domain}:{pport}:127.0.0.1',
'--proxy-cacert', self.env.ca.cert_file,
]
if proto == 'h2':
xargs.append('--proxy-http2')
else:
xargs = [
'--proxy', f'http://{self.env.proxy_domain}:{self.env.proxy_port}/',
'--resolve', f'{self.env.proxy_domain}:{self.env.proxy_port}:127.0.0.1',
]
if tunnel:
xargs.append('--proxytunnel')
return xargs
def http_get(self, url: str, extra_args: Optional[List[str]] = None):
return self._raw(url, options=extra_args, with_stats=False)