romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-08-26 07:23:31 +03:00
Code Issues Projects Releases Packages Wiki Activity

black formatting

Browse source
This commit is contained in:
James Fuller 2025-02-09 17:29:30 +01:00
parent d7333c2bed
commit 53336d9e0b
No known key found for this signature in database
GPG key ID: 62A7A992CAF66B84
36 changed files with 4737 additions and 3515 deletions
Download patch file Download diff file Expand all files Collapse all files

8
tests/http/testenv/__init__.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -25,8 +25,10 @@
###########################################################################
# ruff: noqa: F401, E402
import pytest
pytest.register_assert_rewrite("testenv.env", "testenv.curl", "testenv.caddy",
"testenv.httpd", "testenv.nghttpx")
pytest.register_assert_rewrite(
"testenv.env", "testenv.curl", "testenv.caddy", "testenv.httpd", "testenv.nghttpx"
)
from .env import Env
from .certs import TestCA, Credentials

79
tests/http/testenv/caddy.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -39,15 +39,14 @@ log = logging.getLogger(__name__)
class Caddy:
def __init__(self, env: Env):
self.env = env
self._caddy = os.environ['CADDY'] if 'CADDY' in os.environ else env.caddy
self._caddy_dir = os.path.join(env.gen_dir, 'caddy')
self._docs_dir = os.path.join(self._caddy_dir, 'docs')
self._conf_file = os.path.join(self._caddy_dir, 'Caddyfile')
self._error_log = os.path.join(self._caddy_dir, 'caddy.log')
self._tmp_dir = os.path.join(self._caddy_dir, 'tmp')
self._caddy = os.environ["CADDY"] if "CADDY" in os.environ else env.caddy
self._caddy_dir = os.path.join(env.gen_dir, "caddy")
self._docs_dir = os.path.join(self._caddy_dir, "docs")
self._conf_file = os.path.join(self._caddy_dir, "Caddyfile")
self._error_log = os.path.join(self._caddy_dir, "caddy.log")
self._tmp_dir = os.path.join(self._caddy_dir, "tmp")
self._process = None
self._rmf(self._error_log)
@ -78,11 +77,11 @@ class Caddy:
if self._process:
self.stop()
self._write_config()
args = [
self._caddy, 'run'
]
caddyerr = open(self._error_log, 'a')
self._process = subprocess.Popen(args=args, cwd=self._caddy_dir, stderr=caddyerr)
args = [self._caddy, "run"]
caddyerr = open(self._error_log, "a")
self._process = subprocess.Popen(
args=args, cwd=self._caddy_dir, stderr=caddyerr
)
if self._process.returncode is not None:
return False
return not wait_live or self.wait_live(timeout=timedelta(seconds=5))
@ -109,12 +108,12 @@ class Caddy:
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
check_url = f'https://{self.env.domain1}:{self.port}/'
check_url = f"https://{self.env.domain1}:{self.port}/"
r = curl.http_get(url=check_url)
if r.exit_code != 0:
return True
log.debug(f'waiting for caddy to stop responding: {r}')
time.sleep(.1)
log.debug(f"waiting for caddy to stop responding: {r}")
time.sleep(0.1)
log.debug(f"Server still responding after {timeout}")
return False
@ -122,11 +121,11 @@ class Caddy:
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
check_url = f'https://{self.env.domain1}:{self.port}/'
check_url = f"https://{self.env.domain1}:{self.port}/"
r = curl.http_get(url=check_url)
if r.exit_code == 0:
return True
time.sleep(.1)
time.sleep(0.1)
log.error(f"Caddy still not responding after {timeout}")
return False
@ -147,30 +146,30 @@ class Caddy:
assert creds2 # convince pytype this isn't None
self._mkpath(self._docs_dir)
self._mkpath(self._tmp_dir)
with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd:
with open(os.path.join(self._docs_dir, "data.json"), "w") as fd:
data = {
'server': f'{domain1}',
"server": f"{domain1}",
}
fd.write(JSONEncoder().encode(data))
with open(self._conf_file, 'w') as fd:
conf = [ # base server config
'{',
f' http_port {self.env.caddy_http_port}',
f' https_port {self.env.caddy_https_port}',
f' servers :{self.env.caddy_https_port} {{',
' protocols h3 h2 h1',
' }',
'}',
f'{domain1}:{self.env.caddy_https_port} {{',
' file_server * {',
f' root {self._docs_dir}',
' }',
f' tls {creds1.cert_file} {creds1.pkey_file}',
'}',
f'{domain2} {{',
f' reverse_proxy /* http://localhost:{self.env.http_port} {{',
' }',
f' tls {creds2.cert_file} {creds2.pkey_file}',
'}',
with open(self._conf_file, "w") as fd:
conf = [ # base server config
"{",
f" http_port {self.env.caddy_http_port}",
f" https_port {self.env.caddy_https_port}",
f" servers :{self.env.caddy_https_port} {{",
" protocols h3 h2 h1",
" }",
"}",
f"{domain1}:{self.env.caddy_https_port} {{",
" file_server * {",
f" root {self._docs_dir}",
" }",
f" tls {creds1.cert_file} {creds1.pkey_file}",
"}",
f"{domain2} {{",
f" reverse_proxy /* http://localhost:{self.env.http_port} {{",
" }",
f" tls {creds2.cert_file} {creds2.pkey_file}",
"}",
]
fd.write("\n".join(conf))

446
tests/http/testenv/certs.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -36,52 +36,59 @@ from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import ec, rsa
from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption, load_pem_private_key
from cryptography.hazmat.primitives.serialization import (
Encoding,
PrivateFormat,
NoEncryption,
load_pem_private_key,
)
from cryptography.x509 import ExtendedKeyUsageOID, NameOID
EC_SUPPORTED = {}
EC_SUPPORTED.update([(curve.name.upper(), curve) for curve in [
ec.SECP192R1,
ec.SECP224R1,
ec.SECP256R1,
ec.SECP384R1,
]])
EC_SUPPORTED.update(
[
(curve.name.upper(), curve)
for curve in [
ec.SECP192R1,
ec.SECP224R1,
ec.SECP256R1,
ec.SECP384R1,
]
]
)
def _private_key(key_type):
if isinstance(key_type, str):
key_type = key_type.upper()
m = re.match(r'^(RSA)?(\d+)$', key_type)
m = re.match(r"^(RSA)?(\d+)$", key_type)
if m:
key_type = int(m.group(2))
if isinstance(key_type, int):
return rsa.generate_private_key(
public_exponent=65537,
key_size=key_type,
backend=default_backend()
public_exponent=65537, key_size=key_type, backend=default_backend()
)
if not isinstance(key_type, ec.EllipticCurve) and key_type in EC_SUPPORTED:
key_type = EC_SUPPORTED[key_type]
return ec.generate_private_key(
curve=key_type,
backend=default_backend()
)
return ec.generate_private_key(curve=key_type, backend=default_backend())
class CertificateSpec:
def __init__(self, name: Optional[str] = None,
domains: Optional[List[str]] = None,
email: Optional[str] = None,
key_type: Optional[str] = None,
single_file: bool = False,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
client: bool = False,
check_valid: bool = True,
sub_specs: Optional[List['CertificateSpec']] = None):
def __init__(
self,
name: Optional[str] = None,
domains: Optional[List[str]] = None,
email: Optional[str] = None,
key_type: Optional[str] = None,
single_file: bool = False,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
client: bool = False,
check_valid: bool = True,
sub_specs: Optional[List["CertificateSpec"]] = None,
):
self._name = name
self.domains = domains
self.client = client
@ -113,12 +120,9 @@ class CertificateSpec:
class Credentials:
def __init__(self,
name: str,
cert: Any,
pkey: Any,
issuer: Optional['Credentials'] = None):
def __init__(
self, name: str, cert: Any, pkey: Any, issuer: Optional["Credentials"] = None
):
self._name = name
self._cert = cert
self._pkey = pkey
@ -161,18 +165,25 @@ class Credentials:
def pkey_pem(self) -> bytes:
return self._pkey.private_bytes(
Encoding.PEM,
PrivateFormat.TraditionalOpenSSL if self.key_type.startswith('rsa') else PrivateFormat.PKCS8,
NoEncryption())
PrivateFormat.TraditionalOpenSSL
if self.key_type.startswith("rsa")
else PrivateFormat.PKCS8,
NoEncryption(),
)
@property
def issuer(self) -> Optional['Credentials']:
def issuer(self) -> Optional["Credentials"]:
return self._issuer
def set_store(self, store: 'CertStore'):
def set_store(self, store: "CertStore"):
self._store = store
def set_files(self, cert_file: str, pkey_file: Optional[str] = None,
combined_file: Optional[str] = None):
def set_files(
self,
cert_file: str,
pkey_file: Optional[str] = None,
combined_file: Optional[str] = None,
):
self._cert_file = cert_file
self._pkey_file = pkey_file
self._combined_file = combined_file
@ -189,28 +200,39 @@ class Credentials:
def combined_file(self) -> Optional[str]:
return self._combined_file
def get_first(self, name) -> Optional['Credentials']:
def get_first(self, name) -> Optional["Credentials"]:
creds = self._store.get_credentials_for_name(name) if self._store else []
return creds[0] if len(creds) else None
def get_credentials_for_name(self, name) -> List['Credentials']:
def get_credentials_for_name(self, name) -> List["Credentials"]:
return self._store.get_credentials_for_name(name) if self._store else []
def issue_certs(self, specs: List[CertificateSpec],
chain: Optional[List['Credentials']] = None) -> List['Credentials']:
def issue_certs(
self, specs: List[CertificateSpec], chain: Optional[List["Credentials"]] = None
) -> List["Credentials"]:
return [self.issue_cert(spec=spec, chain=chain) for spec in specs]
def issue_cert(self, spec: CertificateSpec,
chain: Optional[List['Credentials']] = None) -> 'Credentials':
def issue_cert(
self, spec: CertificateSpec, chain: Optional[List["Credentials"]] = None
) -> "Credentials":
key_type = spec.key_type if spec.key_type else self.key_type
creds = None
if self._store:
creds = self._store.load_credentials(
name=spec.name, key_type=key_type, single_file=spec.single_file,
issuer=self, check_valid=spec.check_valid)
name=spec.name,
key_type=key_type,
single_file=spec.single_file,
issuer=self,
check_valid=spec.check_valid,
)
if creds is None:
creds = TestCA.create_credentials(spec=spec, issuer=self, key_type=key_type,
valid_from=spec.valid_from, valid_to=spec.valid_to)
creds = TestCA.create_credentials(
spec=spec,
issuer=self,
key_type=key_type,
valid_from=spec.valid_from,
valid_to=spec.valid_to,
)
if self._store:
self._store.save(creds, single_file=spec.single_file)
if spec.type == "ca":
@ -227,7 +249,6 @@ class Credentials:
class CertStore:
def __init__(self, fpath: str):
self._store_dir = fpath
if not os.path.exists(self._store_dir):
@ -238,9 +259,13 @@ class CertStore:
def path(self) -> str:
return self._store_dir
def save(self, creds: Credentials, name: Optional[str] = None,
chain: Optional[List[Credentials]] = None,
single_file: bool = False) -> None:
def save(
self,
creds: Credentials,
name: Optional[str] = None,
chain: Optional[List[Credentials]] = None,
single_file: bool = False,
) -> None:
name = name if name is not None else creds.name
cert_file = self.get_cert_file(name=name, key_type=creds.key_type)
pkey_file = self.get_pkey_file(name=name, key_type=creds.key_type)
@ -274,7 +299,7 @@ class CertStore:
chain.append(creds)
if not with_root and len(chain) > 1:
chain = chain[:-1]
chain_file = os.path.join(self._store_dir, f'{name}-{infix}.pem')
chain_file = os.path.join(self._store_dir, f"{name}-{infix}.pem")
with open(chain_file, "wb") as fd:
for c in chain:
fd.write(c.cert_pem)
@ -289,14 +314,14 @@ class CertStore:
def get_cert_file(self, name: str, key_type=None) -> str:
key_infix = ".{0}".format(key_type) if key_type is not None else ""
return os.path.join(self._store_dir, f'{name}{key_infix}.cert.pem')
return os.path.join(self._store_dir, f"{name}{key_infix}.cert.pem")
def get_pkey_file(self, name: str, key_type=None) -> str:
key_infix = ".{0}".format(key_type) if key_type is not None else ""
return os.path.join(self._store_dir, f'{name}{key_infix}.pkey.pem')
return os.path.join(self._store_dir, f"{name}{key_infix}.pkey.pem")
def get_combined_file(self, name: str, key_type=None) -> str:
return os.path.join(self._store_dir, f'{name}.pem')
return os.path.join(self._store_dir, f"{name}.pem")
def load_pem_cert(self, fpath: str) -> x509.Certificate:
with open(fpath) as fd:
@ -306,27 +331,36 @@ class CertStore:
with open(fpath) as fd:
return load_pem_private_key("".join(fd.readlines()).encode(), password=None)
def load_credentials(self, name: str, key_type=None,
single_file: bool = False,
issuer: Optional[Credentials] = None,
check_valid: bool = False):
def load_credentials(
self,
name: str,
key_type=None,
single_file: bool = False,
issuer: Optional[Credentials] = None,
check_valid: bool = False,
):
cert_file = self.get_cert_file(name=name, key_type=key_type)
pkey_file = cert_file if single_file else self.get_pkey_file(name=name, key_type=key_type)
pkey_file = (
cert_file
if single_file
else self.get_pkey_file(name=name, key_type=key_type)
)
comb_file = self.get_combined_file(name=name, key_type=key_type)
if os.path.isfile(cert_file) and os.path.isfile(pkey_file):
cert = self.load_pem_cert(cert_file)
pkey = self.load_pem_pkey(pkey_file)
try:
now = datetime.now(tz=timezone.utc)
if check_valid and \
((cert.not_valid_after_utc < now) or
(cert.not_valid_before_utc > now)):
if check_valid and (
(cert.not_valid_after_utc < now)
or (cert.not_valid_before_utc > now)
):
return None
except AttributeError: # older python
now = datetime.now()
if check_valid and \
((cert.not_valid_after < now) or
(cert.not_valid_before > now)):
if check_valid and (
(cert.not_valid_after < now) or (cert.not_valid_before > now)
):
return None
creds = Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer)
creds.set_store(self)
@ -337,9 +371,10 @@ class CertStore:
class TestCA:
@classmethod
def create_root(cls, name: str, store_dir: str, key_type: str = "rsa2048") -> Credentials:
def create_root(
cls, name: str, store_dir: str, key_type: str = "rsa2048"
) -> Credentials:
store = CertStore(fpath=store_dir)
creds = store.load_credentials(name="ca", key_type=key_type, issuer=None)
if creds is None:
@ -349,36 +384,61 @@ class TestCA:
return creds
@staticmethod
def create_credentials(spec: CertificateSpec, issuer: Credentials, key_type: Any,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
def create_credentials(
spec: CertificateSpec,
issuer: Credentials,
key_type: Any,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
"""
Create a certificate signed by this CA for the given domains.
:returns: the certificate and private key PEM file paths
"""
if spec.domains and len(spec.domains):
creds = TestCA._make_server_credentials(name=spec.name, domains=spec.domains,
issuer=issuer, valid_from=valid_from,
valid_to=valid_to, key_type=key_type)
creds = TestCA._make_server_credentials(
name=spec.name,
domains=spec.domains,
issuer=issuer,
valid_from=valid_from,
valid_to=valid_to,
key_type=key_type,
)
elif spec.client:
creds = TestCA._make_client_credentials(name=spec.name, issuer=issuer,
email=spec.email, valid_from=valid_from,
valid_to=valid_to, key_type=key_type)
creds = TestCA._make_client_credentials(
name=spec.name,
issuer=issuer,
email=spec.email,
valid_from=valid_from,
valid_to=valid_to,
key_type=key_type,
)
elif spec.name:
creds = TestCA._make_ca_credentials(name=spec.name, issuer=issuer,
valid_from=valid_from, valid_to=valid_to,
key_type=key_type)
creds = TestCA._make_ca_credentials(
name=spec.name,
issuer=issuer,
valid_from=valid_from,
valid_to=valid_to,
key_type=key_type,
)
else:
raise Exception(f"unrecognized certificate specification: {spec}")
return creds
@staticmethod
def _make_x509_name(org_name: Optional[str] = None, common_name: Optional[str] = None, parent: x509.Name = None) -> x509.Name:
def _make_x509_name(
org_name: Optional[str] = None,
common_name: Optional[str] = None,
parent: x509.Name = None,
) -> x509.Name:
name_pieces = []
if org_name:
oid = NameOID.ORGANIZATIONAL_UNIT_NAME if parent else NameOID.ORGANIZATION_NAME
oid = (
NameOID.ORGANIZATIONAL_UNIT_NAME
if parent
else NameOID.ORGANIZATION_NAME
)
name_pieces.append(x509.NameAttribute(oid, org_name))
elif common_name:
name_pieces.append(x509.NameAttribute(NameOID.COMMON_NAME, common_name))
@ -388,11 +448,11 @@ class TestCA:
@staticmethod
def _make_csr(
subject: x509.Name,
pkey: Any,
issuer_subject: Optional[Credentials],
valid_from_delta: Optional[timedelta] = None,
valid_until_delta: Optional[timedelta] = None
subject: x509.Name,
pkey: Any,
issuer_subject: Optional[Credentials],
valid_from_delta: Optional[timedelta] = None,
valid_until_delta: Optional[timedelta] = None,
):
pubkey = pkey.public_key()
issuer_subject = issuer_subject if issuer_subject is not None else subject
@ -420,28 +480,35 @@ class TestCA:
@staticmethod
def _add_ca_usages(csr: Any) -> Any:
return csr.add_extension(
x509.BasicConstraints(ca=True, path_length=9),
critical=True,
).add_extension(
x509.KeyUsage(
digital_signature=True,
content_commitment=False,
key_encipherment=False,
data_encipherment=False,
key_agreement=False,
key_cert_sign=True,
crl_sign=True,
encipher_only=False,
decipher_only=False),
critical=True
).add_extension(
x509.ExtendedKeyUsage([
ExtendedKeyUsageOID.CLIENT_AUTH,
ExtendedKeyUsageOID.SERVER_AUTH,
ExtendedKeyUsageOID.CODE_SIGNING,
]),
critical=True
return (
csr.add_extension(
x509.BasicConstraints(ca=True, path_length=9),
critical=True,
)
.add_extension(
x509.KeyUsage(
digital_signature=True,
content_commitment=False,
key_encipherment=False,
data_encipherment=False,
key_agreement=False,
key_cert_sign=True,
crl_sign=True,
encipher_only=False,
decipher_only=False,
),
critical=True,
)
.add_extension(
x509.ExtendedKeyUsage(
[
ExtendedKeyUsageOID.CLIENT_AUTH,
ExtendedKeyUsageOID.SERVER_AUTH,
ExtendedKeyUsageOID.CODE_SIGNING,
]
),
critical=True,
)
)
@staticmethod
@ -454,33 +521,47 @@ class TestCA:
except: # noqa: E722
names.append(x509.DNSName(name))
return csr.add_extension(
x509.BasicConstraints(ca=False, path_length=None),
critical=True,
).add_extension(
x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier(
issuer.certificate.extensions.get_extension_for_class(
x509.SubjectKeyIdentifier).value),
critical=False
).add_extension(
x509.SubjectAlternativeName(names), critical=True,
).add_extension(
x509.ExtendedKeyUsage([
ExtendedKeyUsageOID.SERVER_AUTH,
]),
critical=False
return (
csr.add_extension(
x509.BasicConstraints(ca=False, path_length=None),
critical=True,
)
.add_extension(
x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier(
issuer.certificate.extensions.get_extension_for_class(
x509.SubjectKeyIdentifier
).value
),
critical=False,
)
.add_extension(
x509.SubjectAlternativeName(names),
critical=True,
)
.add_extension(
x509.ExtendedKeyUsage(
[
ExtendedKeyUsageOID.SERVER_AUTH,
]
),
critical=False,
)
)
@staticmethod
def _add_client_usages(csr: Any, issuer: Credentials, rfc82name: Optional[str] = None) -> Any:
def _add_client_usages(
csr: Any, issuer: Credentials, rfc82name: Optional[str] = None
) -> Any:
cert = csr.add_extension(
x509.BasicConstraints(ca=False, path_length=None),
critical=True,
).add_extension(
x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier(
issuer.certificate.extensions.get_extension_for_class(
x509.SubjectKeyIdentifier).value),
critical=False
x509.SubjectKeyIdentifier
).value
),
critical=False,
)
if rfc82name:
cert.add_extension(
@ -488,19 +569,23 @@ class TestCA:
critical=True,
)
cert.add_extension(
x509.ExtendedKeyUsage([
ExtendedKeyUsageOID.CLIENT_AUTH,
]),
critical=True
x509.ExtendedKeyUsage(
[
ExtendedKeyUsageOID.CLIENT_AUTH,
]
),
critical=True,
)
return cert
@staticmethod
def _make_ca_credentials(name, key_type: Any,
issuer: Optional[Credentials] = None,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
def _make_ca_credentials(
name,
key_type: Any,
issuer: Optional[Credentials] = None,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
pkey = _private_key(key_type=key_type)
if issuer is not None:
issuer_subject = issuer.certificate.subject
@ -508,47 +593,70 @@ class TestCA:
else:
issuer_subject = None
issuer_key = pkey
subject = TestCA._make_x509_name(org_name=name, parent=issuer.subject if issuer else None)
csr = TestCA._make_csr(subject=subject,
issuer_subject=issuer_subject, pkey=pkey,
valid_from_delta=valid_from, valid_until_delta=valid_to)
subject = TestCA._make_x509_name(
org_name=name, parent=issuer.subject if issuer else None
)
csr = TestCA._make_csr(
subject=subject,
issuer_subject=issuer_subject,
pkey=pkey,
valid_from_delta=valid_from,
valid_until_delta=valid_to,
)
csr = TestCA._add_ca_usages(csr)
cert = csr.sign(private_key=issuer_key,
algorithm=hashes.SHA256(),
backend=default_backend())
cert = csr.sign(
private_key=issuer_key, algorithm=hashes.SHA256(), backend=default_backend()
)
return Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer)
@staticmethod
def _make_server_credentials(name: str, domains: List[str], issuer: Credentials,
key_type: Any,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
def _make_server_credentials(
name: str,
domains: List[str],
issuer: Credentials,
key_type: Any,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
pkey = _private_key(key_type=key_type)
subject = TestCA._make_x509_name(common_name=name, parent=issuer.subject)
csr = TestCA._make_csr(subject=subject,
issuer_subject=issuer.certificate.subject, pkey=pkey,
valid_from_delta=valid_from, valid_until_delta=valid_to)
csr = TestCA._make_csr(
subject=subject,
issuer_subject=issuer.certificate.subject,
pkey=pkey,
valid_from_delta=valid_from,
valid_until_delta=valid_to,
)
csr = TestCA._add_leaf_usages(csr, domains=domains, issuer=issuer)
cert = csr.sign(private_key=issuer.private_key,
algorithm=hashes.SHA256(),
backend=default_backend())
cert = csr.sign(
private_key=issuer.private_key,
algorithm=hashes.SHA256(),
backend=default_backend(),
)
return Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer)
@staticmethod
def _make_client_credentials(name: str,
issuer: Credentials, email: Optional[str],
key_type: Any,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
def _make_client_credentials(
name: str,
issuer: Credentials,
email: Optional[str],
key_type: Any,
valid_from: timedelta = timedelta(days=-1),
valid_to: timedelta = timedelta(days=89),
) -> Credentials:
pkey = _private_key(key_type=key_type)
subject = TestCA._make_x509_name(common_name=name, parent=issuer.subject)
csr = TestCA._make_csr(subject=subject,
issuer_subject=issuer.certificate.subject, pkey=pkey,
valid_from_delta=valid_from, valid_until_delta=valid_to)
csr = TestCA._make_csr(
subject=subject,
issuer_subject=issuer.certificate.subject,
pkey=pkey,
valid_from_delta=valid_from,
valid_until_delta=valid_to,
)
csr = TestCA._add_client_usages(csr, issuer=issuer, rfc82name=email)
cert = csr.sign(private_key=issuer.private_key,
algorithm=hashes.SHA256(),
backend=default_backend())
cert = csr.sign(
private_key=issuer.private_key,
algorithm=hashes.SHA256(),
backend=default_backend(),
)
return Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer)

67
tests/http/testenv/client.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -39,19 +39,23 @@ log = logging.getLogger(__name__)
class LocalClient:
def __init__(self, name: str, env: Env, run_dir: Optional[str] = None,
timeout: Optional[float] = None,
run_env: Optional[Dict[str,str]] = None):
def __init__(
self,
name: str,
env: Env,
run_dir: Optional[str] = None,
timeout: Optional[float] = None,
run_env: Optional[Dict[str, str]] = None,
):
self.name = name
self.path = os.path.join(env.build_dir, f'tests/http/clients/{name}')
self.path = os.path.join(env.build_dir, f"tests/http/clients/{name}")
self.env = env
self._run_env = run_env
self._timeout = timeout if timeout else env.test_timeout
self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl
self._curl = os.environ["CURL"] if "CURL" in os.environ else env.curl
self._run_dir = run_dir if run_dir else os.path.join(env.gen_dir, name)
self._stdoutfile = f'{self._run_dir}/stdout'
self._stderrfile = f'{self._run_dir}/stderr'
self._stdoutfile = f"{self._run_dir}/stdout"
self._stderrfile = f"{self._run_dir}/stderr"
self._rmrf(self._run_dir)
self._mkpath(self._run_dir)
@ -67,7 +71,7 @@ class LocalClient:
return os.path.exists(self.path)
def download_file(self, i: int) -> str:
return os.path.join(self._run_dir, f'download_{i}.data')
return os.path.join(self._run_dir, f"download_{i}.data")
def _rmf(self, path):
if os.path.exists(path):
@ -91,31 +95,44 @@ class LocalClient:
run_env = None
if self._run_env:
run_env = self._run_env.copy()
for key in ['CURL_DEBUG']:
for key in ["CURL_DEBUG"]:
if key in os.environ and key not in run_env:
run_env[key] = os.environ[key]
try:
with open(self._stdoutfile, 'w') as cout, open(self._stderrfile, 'w') as cerr:
p = subprocess.run(myargs, stderr=cerr, stdout=cout,
cwd=self._run_dir, shell=False,
input=None, env=run_env,
timeout=self._timeout)
with open(self._stdoutfile, "w") as cout, open(
self._stderrfile, "w"
) as cerr:
p = subprocess.run(
myargs,
stderr=cerr,
stdout=cout,
cwd=self._run_dir,
shell=False,
input=None,
env=run_env,
timeout=self._timeout,
)
exitcode = p.returncode
except subprocess.TimeoutExpired:
log.warning(f'Timeout after {self._timeout}s: {args}')
log.warning(f"Timeout after {self._timeout}s: {args}")
exitcode = -1
exception = 'TimeoutExpired'
exception = "TimeoutExpired"
coutput = open(self._stdoutfile).readlines()
cerrput = open(self._stderrfile).readlines()
return ExecResult(args=myargs, exit_code=exitcode, exception=exception,
stdout=coutput, stderr=cerrput,
duration=datetime.now() - start)
return ExecResult(
args=myargs,
exit_code=exitcode,
exception=exception,
stdout=coutput,
stderr=cerrput,
duration=datetime.now() - start,
)
def dump_logs(self):
lines = []
lines.append('>>--stdout ----------------------------------------------\n')
lines.append(">>--stdout ----------------------------------------------\n")
lines.extend(open(self._stdoutfile).readlines())
lines.append('>>--stderr ----------------------------------------------\n')
lines.append(">>--stderr ----------------------------------------------\n")
lines.extend(open(self._stderrfile).readlines())
lines.append('<<-------------------------------------------------------\n')
return ''.join(lines)
lines.append("<<-------------------------------------------------------\n")
return "".join(lines)

1033
tests/http/testenv/curl.py
View file

File diff suppressed because it is too large Load diff

334
tests/http/testenv/env.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -52,92 +52,93 @@ def init_config_from(conf_path):
TESTS_HTTPD_PATH = os.path.dirname(os.path.dirname(__file__))
TOP_PATH = os.path.join(os.getcwd(), os.path.pardir)
DEF_CONFIG = init_config_from(os.path.join(TOP_PATH, 'tests', 'http', 'config.ini'))
CURL = os.path.join(TOP_PATH, 'src', 'curl')
DEF_CONFIG = init_config_from(os.path.join(TOP_PATH, "tests", "http", "config.ini"))
CURL = os.path.join(TOP_PATH, "src", "curl")
class EnvConfig:
def __init__(self):
self.tests_dir = TESTS_HTTPD_PATH
self.gen_dir = os.path.join(self.tests_dir, 'gen')
self.gen_dir = os.path.join(self.tests_dir, "gen")
self.project_dir = os.path.dirname(os.path.dirname(self.tests_dir))
self.build_dir = TOP_PATH
self.config = DEF_CONFIG
# check cur and its features
self.curl = CURL
if 'CURL' in os.environ:
self.curl = os.environ['CURL']
if "CURL" in os.environ:
self.curl = os.environ["CURL"]
self.curl_props = {
'version_string': '',
'version': '',
'os': '',
'fullname': '',
'features_string': '',
'features': set(),
'protocols_string': '',
'protocols': set(),
'libs': set(),
'lib_versions': set(),
"version_string": "",
"version": "",
"os": "",
"fullname": "",
"features_string": "",
"features": set(),
"protocols_string": "",
"protocols": set(),
"libs": set(),
"lib_versions": set(),
}
self.curl_is_debug = False
self.curl_protos = []
p = subprocess.run(args=[self.curl, '-V'],
capture_output=True, text=True)
p = subprocess.run(args=[self.curl, "-V"], capture_output=True, text=True)
if p.returncode != 0:
raise RuntimeError(f'{self.curl} -V failed with exit code: {p.returncode}')
if p.stderr.startswith('WARNING:'):
raise RuntimeError(f"{self.curl} -V failed with exit code: {p.returncode}")
if p.stderr.startswith("WARNING:"):
self.curl_is_debug = True
for line in p.stdout.splitlines(keepends=False):
if line.startswith('curl '):
self.curl_props['version_string'] = line
m = re.match(r'^curl (?P<version>\S+) (?P<os>\S+) (?P<libs>.*)$', line)
if line.startswith("curl "):
self.curl_props["version_string"] = line
m = re.match(r"^curl (?P<version>\S+) (?P<os>\S+) (?P<libs>.*)$", line)
if m:
self.curl_props['fullname'] = m.group(0)
self.curl_props['version'] = m.group('version')
self.curl_props['os'] = m.group('os')
self.curl_props['lib_versions'] = {
lib.lower() for lib in m.group('libs').split(' ')
self.curl_props["fullname"] = m.group(0)
self.curl_props["version"] = m.group("version")
self.curl_props["os"] = m.group("os")
self.curl_props["lib_versions"] = {
lib.lower() for lib in m.group("libs").split(" ")
}
self.curl_props['libs'] = {
re.sub(r'/[a-z0-9.-]*', '', lib) for lib in self.curl_props['lib_versions']
self.curl_props["libs"] = {
re.sub(r"/[a-z0-9.-]*", "", lib)
for lib in self.curl_props["lib_versions"]
}
if line.startswith('Features: '):
self.curl_props['features_string'] = line[10:]
self.curl_props['features'] = {
feat.lower() for feat in line[10:].split(' ')
if line.startswith("Features: "):
self.curl_props["features_string"] = line[10:]
self.curl_props["features"] = {
feat.lower() for feat in line[10:].split(" ")
}
if line.startswith('Protocols: '):
self.curl_props['protocols_string'] = line[11:]
self.curl_props['protocols'] = {
prot.lower() for prot in line[11:].split(' ')
if line.startswith("Protocols: "):
self.curl_props["protocols_string"] = line[11:]
self.curl_props["protocols"] = {
prot.lower() for prot in line[11:].split(" ")
}
self.ports = alloc_ports(port_specs={
'ftp': socket.SOCK_STREAM,
'ftps': socket.SOCK_STREAM,
'http': socket.SOCK_STREAM,
'https': socket.SOCK_STREAM,
'nghttpx_https': socket.SOCK_STREAM,
'proxy': socket.SOCK_STREAM,
'proxys': socket.SOCK_STREAM,
'h2proxys': socket.SOCK_STREAM,
'caddy': socket.SOCK_STREAM,
'caddys': socket.SOCK_STREAM,
'ws': socket.SOCK_STREAM,
})
self.httpd = self.config['httpd']['httpd']
self.apxs = self.config['httpd']['apxs']
self.ports = alloc_ports(
port_specs={
"ftp": socket.SOCK_STREAM,
"ftps": socket.SOCK_STREAM,
"http": socket.SOCK_STREAM,
"https": socket.SOCK_STREAM,
"nghttpx_https": socket.SOCK_STREAM,
"proxy": socket.SOCK_STREAM,
"proxys": socket.SOCK_STREAM,
"h2proxys": socket.SOCK_STREAM,
"caddy": socket.SOCK_STREAM,
"caddys": socket.SOCK_STREAM,
"ws": socket.SOCK_STREAM,
}
)
self.httpd = self.config["httpd"]["httpd"]
self.apxs = self.config["httpd"]["apxs"]
if len(self.apxs) == 0:
self.apxs = None
self._httpd_version = None
self.examples_pem = {
'key': 'xxx',
'cert': 'xxx',
"key": "xxx",
"cert": "xxx",
}
self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs')
self.tld = 'http.curl.se'
self.htdocs_dir = os.path.join(self.gen_dir, "htdocs")
self.tld = "http.curl.se"
self.domain1 = f"one.{self.tld}"
self.domain1brotli = f"brotli.one.{self.tld}"
self.domain2 = f"two.{self.tld}"
@ -145,60 +146,82 @@ class EnvConfig:
self.proxy_domain = f"proxy.{self.tld}"
self.expired_domain = f"expired.{self.tld}"
self.cert_specs = [
CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost', '127.0.0.1'], key_type='rsa2048'),
CertificateSpec(domains=[self.domain2], key_type='rsa2048'),
CertificateSpec(domains=[self.ftp_domain], key_type='rsa2048'),
CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'),
CertificateSpec(domains=[self.expired_domain], key_type='rsa2048',
valid_from=timedelta(days=-100), valid_to=timedelta(days=-10)),
CertificateSpec(name="clientsX", sub_specs=[
CertificateSpec(name="user1", client=True),
]),
CertificateSpec(
domains=[self.domain1, self.domain1brotli, "localhost", "127.0.0.1"],
key_type="rsa2048",
),
CertificateSpec(domains=[self.domain2], key_type="rsa2048"),
CertificateSpec(domains=[self.ftp_domain], key_type="rsa2048"),
CertificateSpec(
domains=[self.proxy_domain, "127.0.0.1"], key_type="rsa2048"
),
CertificateSpec(
domains=[self.expired_domain],
key_type="rsa2048",
valid_from=timedelta(days=-100),
valid_to=timedelta(days=-10),
),
CertificateSpec(
name="clientsX",
sub_specs=[
CertificateSpec(name="user1", client=True),
],
),
]
self.nghttpx = self.config['nghttpx']['nghttpx']
self.nghttpx = self.config["nghttpx"]["nghttpx"]
if len(self.nghttpx.strip()) == 0:
self.nghttpx = None
self._nghttpx_version = None
self.nghttpx_with_h3 = False
if self.nghttpx is not None:
p = subprocess.run(args=[self.nghttpx, '-v'],
capture_output=True, text=True)
p = subprocess.run(
args=[self.nghttpx, "-v"], capture_output=True, text=True
)
if p.returncode != 0:
# not a working nghttpx
self.nghttpx = None
else:
self._nghttpx_version = re.sub(r'^nghttpx\s*', '', p.stdout.strip())
self.nghttpx_with_h3 = re.match(r'.* nghttp3/.*', p.stdout.strip()) is not None
log.debug(f'nghttpx -v: {p.stdout}')
self._nghttpx_version = re.sub(r"^nghttpx\s*", "", p.stdout.strip())
self.nghttpx_with_h3 = (
re.match(r".* nghttp3/.*", p.stdout.strip()) is not None
)
log.debug(f"nghttpx -v: {p.stdout}")
self.caddy = self.config['caddy']['caddy']
self.caddy = self.config["caddy"]["caddy"]
self._caddy_version = None
if len(self.caddy.strip()) == 0:
self.caddy = None
if self.caddy is not None:
try:
p = subprocess.run(args=[self.caddy, 'version'],
capture_output=True, text=True)
p = subprocess.run(
args=[self.caddy, "version"], capture_output=True, text=True
)
if p.returncode != 0:
# not a working caddy
self.caddy = None
m = re.match(r'v?(\d+\.\d+\.\d+).*', p.stdout)
m = re.match(r"v?(\d+\.\d+\.\d+).*", p.stdout)
if m:
self._caddy_version = m.group(1)
else:
raise RuntimeError(f'Unable to determine cadd version from: {p.stdout}')
raise RuntimeError(
f"Unable to determine cadd version from: {p.stdout}"
)
# TODO: specify specific exceptions here
except: # noqa: E722
self.caddy = None
self.vsftpd = self.config['vsftpd']['vsftpd']
self.vsftpd = self.config["vsftpd"]["vsftpd"]
self._vsftpd_version = None
if self.vsftpd is not None:
try:
with tempfile.TemporaryFile('w+') as tmp:
p = subprocess.run(args=[self.vsftpd, '-v'],
capture_output=True, text=True, stdin=tmp)
with tempfile.TemporaryFile("w+") as tmp:
p = subprocess.run(
args=[self.vsftpd, "-v"],
capture_output=True,
text=True,
stdin=tmp,
)
if p.returncode != 0:
# not a working vsftpd
self.vsftpd = None
@ -211,36 +234,41 @@ class EnvConfig:
# any data there instead.
tmp.seek(0)
ver_text = tmp.read()
m = re.match(r'vsftpd: version (\d+\.\d+\.\d+)', ver_text)
m = re.match(r"vsftpd: version (\d+\.\d+\.\d+)", ver_text)
if m:
self._vsftpd_version = m.group(1)
elif len(p.stderr) == 0:
# vsftp does not use stdout or stderr for printing its version... -.-
self._vsftpd_version = 'unknown'
self._vsftpd_version = "unknown"
else:
raise Exception(f'Unable to determine VsFTPD version from: {p.stderr}')
raise Exception(
f"Unable to determine VsFTPD version from: {p.stderr}"
)
except Exception:
self.vsftpd = None
self._tcpdump = shutil.which('tcpdump')
self._tcpdump = shutil.which("tcpdump")
@property
def httpd_version(self):
if self._httpd_version is None and self.apxs is not None:
try:
p = subprocess.run(args=[self.apxs, '-q', 'HTTPD_VERSION'],
capture_output=True, text=True)
p = subprocess.run(
args=[self.apxs, "-q", "HTTPD_VERSION"],
capture_output=True,
text=True,
)
if p.returncode != 0:
log.error(f'{self.apxs} failed to query HTTPD_VERSION: {p}')
log.error(f"{self.apxs} failed to query HTTPD_VERSION: {p}")
else:
self._httpd_version = p.stdout.strip()
except Exception:
log.exception(f'{self.apxs} failed to run')
log.exception(f"{self.apxs} failed to run")
return self._httpd_version
def versiontuple(self, v):
v = re.sub(r'(\d+\.\d+(\.\d+)?)(-\S+)?', r'\1', v)
return tuple(map(int, v.split('.')))
v = re.sub(r"(\d+\.\d+(\.\d+)?)(-\S+)?", r"\1", v)
return tuple(map(int, v.split(".")))
def httpd_is_at_least(self, minv):
if self.httpd_version is None:
@ -255,15 +283,17 @@ class EnvConfig:
return hv >= self.versiontuple(minv)
def is_complete(self) -> bool:
return os.path.isfile(self.httpd) and \
self.apxs is not None and \
os.path.isfile(self.apxs)
return (
os.path.isfile(self.httpd)
and self.apxs is not None
and os.path.isfile(self.apxs)
)
def get_incomplete_reason(self) -> Optional[str]:
if self.httpd is None or len(self.httpd.strip()) == 0:
return 'httpd not configured, see `--with-test-httpd=<path>`'
return "httpd not configured, see `--with-test-httpd=<path>`"
if not os.path.isfile(self.httpd):
return f'httpd ({self.httpd}) not found'
return f"httpd ({self.httpd}) not found"
if self.apxs is None:
return "command apxs not found (commonly provided in apache2-dev)"
if not os.path.isfile(self.apxs):
@ -288,7 +318,6 @@ class EnvConfig:
class Env:
CONFIG = EnvConfig()
@staticmethod
@ -309,73 +338,74 @@ class Env:
@staticmethod
def have_ssl_curl() -> bool:
return Env.curl_has_feature('ssl') or Env.curl_has_feature('multissl')
return Env.curl_has_feature("ssl") or Env.curl_has_feature("multissl")
@staticmethod
def have_h2_curl() -> bool:
return 'http2' in Env.CONFIG.curl_props['features']
return "http2" in Env.CONFIG.curl_props["features"]
@staticmethod
def have_h3_curl() -> bool:
return 'http3' in Env.CONFIG.curl_props['features']
return "http3" in Env.CONFIG.curl_props["features"]
@staticmethod
def curl_uses_lib(libname: str) -> bool:
return libname.lower() in Env.CONFIG.curl_props['libs']
return libname.lower() in Env.CONFIG.curl_props["libs"]
@staticmethod
def curl_uses_ossl_quic() -> bool:
if Env.have_h3_curl():
return not Env.curl_uses_lib('ngtcp2') and Env.curl_uses_lib('nghttp3')
return not Env.curl_uses_lib("ngtcp2") and Env.curl_uses_lib("nghttp3")
return False
@staticmethod
def curl_version_string() -> str:
return Env.CONFIG.curl_props['version_string']
return Env.CONFIG.curl_props["version_string"]
@staticmethod
def curl_features_string() -> str:
return Env.CONFIG.curl_props['features_string']
return Env.CONFIG.curl_props["features_string"]
@staticmethod
def curl_has_feature(feature: str) -> bool:
return feature.lower() in Env.CONFIG.curl_props['features']
return feature.lower() in Env.CONFIG.curl_props["features"]
@staticmethod
def curl_protocols_string() -> str:
return Env.CONFIG.curl_props['protocols_string']
return Env.CONFIG.curl_props["protocols_string"]
@staticmethod
def curl_has_protocol(protocol: str) -> bool:
return protocol.lower() in Env.CONFIG.curl_props['protocols']
return protocol.lower() in Env.CONFIG.curl_props["protocols"]
@staticmethod
def curl_lib_version(libname: str) -> str:
prefix = f'{libname.lower()}/'
for lversion in Env.CONFIG.curl_props['lib_versions']:
prefix = f"{libname.lower()}/"
for lversion in Env.CONFIG.curl_props["lib_versions"]:
if lversion.startswith(prefix):
return lversion[len(prefix):]
return 'unknown'
return lversion[len(prefix) :]
return "unknown"
@staticmethod
def curl_lib_version_at_least(libname: str, min_version) -> bool:
lversion = Env.curl_lib_version(libname)
if lversion != 'unknown':
return Env.CONFIG.versiontuple(min_version) <= \
Env.CONFIG.versiontuple(lversion)
if lversion != "unknown":
return Env.CONFIG.versiontuple(min_version) <= Env.CONFIG.versiontuple(
lversion
)
return False
@staticmethod
def curl_os() -> str:
return Env.CONFIG.curl_props['os']
return Env.CONFIG.curl_props["os"]
@staticmethod
def curl_fullname() -> str:
return Env.CONFIG.curl_props['fullname']
return Env.CONFIG.curl_props["fullname"]
@staticmethod
def curl_version() -> str:
return Env.CONFIG.curl_props['version']
return Env.CONFIG.curl_props["version"]
@staticmethod
def curl_is_debug() -> bool:
@ -422,17 +452,16 @@ class Env:
return Env.CONFIG.tcpdmp
def __init__(self, pytestconfig=None):
self._verbose = pytestconfig.option.verbose \
if pytestconfig is not None else 0
self._verbose = pytestconfig.option.verbose if pytestconfig is not None else 0
self._ca = None
self._test_timeout = 300.0 if self._verbose > 1 else 60.0 # seconds
def issue_certs(self):
if self._ca is None:
ca_dir = os.path.join(self.CONFIG.gen_dir, 'ca')
self._ca = TestCA.create_root(name=self.CONFIG.tld,
store_dir=ca_dir,
key_type="rsa2048")
ca_dir = os.path.join(self.CONFIG.gen_dir, "ca")
self._ca = TestCA.create_root(
name=self.CONFIG.tld, store_dir=ca_dir, key_type="rsa2048"
)
self._ca.issue_certs(self.CONFIG.cert_specs)
def setup(self):
@ -508,15 +537,15 @@ class Env:
@property
def http_port(self) -> int:
return self.CONFIG.ports['http']
return self.CONFIG.ports["http"]
@property
def https_port(self) -> int:
return self.CONFIG.ports['https']
return self.CONFIG.ports["https"]
@property
def nghttpx_https_port(self) -> int:
return self.CONFIG.ports['nghttpx_https']
return self.CONFIG.ports["nghttpx_https"]
@property
def h3_port(self) -> int:
@ -524,27 +553,27 @@ class Env:
@property
def proxy_port(self) -> int:
return self.CONFIG.ports['proxy']
return self.CONFIG.ports["proxy"]
@property
def proxys_port(self) -> int:
return self.CONFIG.ports['proxys']
return self.CONFIG.ports["proxys"]
@property
def ftp_port(self) -> int:
return self.CONFIG.ports['ftp']
return self.CONFIG.ports["ftp"]
@property
def ftps_port(self) -> int:
return self.CONFIG.ports['ftps']
return self.CONFIG.ports["ftps"]
@property
def h2proxys_port(self) -> int:
return self.CONFIG.ports['h2proxys']
return self.CONFIG.ports["h2proxys"]
def pts_port(self, proto: str = 'http/1.1') -> int:
def pts_port(self, proto: str = "http/1.1") -> int:
# proxy tunnel port
return self.CONFIG.ports['h2proxys' if proto == 'h2' else 'proxys']
return self.CONFIG.ports["h2proxys" if proto == "h2" else "proxys"]
@property
def caddy(self) -> str:
@ -552,11 +581,11 @@ class Env:
@property
def caddy_https_port(self) -> int:
return self.CONFIG.ports['caddys']
return self.CONFIG.ports["caddys"]
@property
def caddy_http_port(self) -> int:
return self.CONFIG.ports['caddy']
return self.CONFIG.ports["caddy"]
@property
def vsftpd(self) -> str:
@ -564,7 +593,7 @@ class Env:
@property
def ws_port(self) -> int:
return self.CONFIG.ports['ws']
return self.CONFIG.ports["ws"]
@property
def curl(self) -> str:
@ -584,37 +613,44 @@ class Env:
@property
def slow_network(self) -> bool:
return "CURL_DBG_SOCK_WBLOCK" in os.environ or \
"CURL_DBG_SOCK_WPARTIAL" in os.environ
return (
"CURL_DBG_SOCK_WBLOCK" in os.environ
or "CURL_DBG_SOCK_WPARTIAL" in os.environ
)
@property
def ci_run(self) -> bool:
return "CURL_CI" in os.environ
def port_for(self, alpn_proto: Optional[str] = None):
if alpn_proto is None or \
alpn_proto in ['h2', 'http/1.1', 'http/1.0', 'http/0.9']:
if alpn_proto is None or alpn_proto in [
"h2",
"http/1.1",
"http/1.0",
"http/0.9",
]:
return self.https_port
if alpn_proto in ['h3']:
if alpn_proto in ["h3"]:
return self.h3_port
return self.http_port
def authority_for(self, domain: str, alpn_proto: Optional[str] = None):
return f'{domain}:{self.port_for(alpn_proto=alpn_proto)}'
return f"{domain}:{self.port_for(alpn_proto=alpn_proto)}"
def make_data_file(self, indir: str, fname: str, fsize: int,
line_length: int = 1024) -> str:
def make_data_file(
self, indir: str, fname: str, fsize: int, line_length: int = 1024
) -> str:
if line_length < 11:
raise RuntimeError('line_length less than 11 not supported')
raise RuntimeError("line_length less than 11 not supported")
fpath = os.path.join(indir, fname)
s10 = "0123456789"
s = round((line_length / 10) + 1) * s10
s = s[0:line_length-11]
with open(fpath, 'w') as fd:
s = s[0 : line_length - 11]
with open(fpath, "w") as fd:
for i in range(int(fsize / line_length)):
fd.write(f"{i:09d}-{s}\n")
remain = int(fsize % line_length)
if remain != 0:
i = int(fsize / line_length) + 1
fd.write(f"{i:09d}-{s}"[0:remain-1] + "\n")
fd.write(f"{i:09d}-{s}"[0 : remain - 1] + "\n")
return fpath

581
tests/http/testenv/httpd.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -42,53 +42,71 @@ log = logging.getLogger(__name__)
class Httpd:
MODULES = [
'log_config', 'logio', 'unixd', 'version', 'watchdog',
'authn_core', 'authn_file',
'authz_user', 'authz_core', 'authz_host',
'auth_basic', 'auth_digest',
'alias', 'env', 'filter', 'headers', 'mime', 'setenvif',
'socache_shmcb',
'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect',
'brotli',
'mpm_event',
"log_config",
"logio",
"unixd",
"version",
"watchdog",
"authn_core",
"authn_file",
"authz_user",
"authz_core",
"authz_host",
"auth_basic",
"auth_digest",
"alias",
"env",
"filter",
"headers",
"mime",
"setenvif",
"socache_shmcb",
"rewrite",
"http2",
"ssl",
"proxy",
"proxy_http",
"proxy_connect",
"brotli",
"mpm_event",
]
COMMON_MODULES_DIRS = [
'/usr/lib/apache2/modules', # debian
'/usr/libexec/apache2/', # macos
"/usr/lib/apache2/modules", # debian
"/usr/libexec/apache2/", # macos
]
MOD_CURLTEST = None
def __init__(self, env: Env, proxy_auth: bool = False):
self.env = env
self._apache_dir = os.path.join(env.gen_dir, 'apache')
self._run_dir = os.path.join(self._apache_dir, 'run')
self._lock_dir = os.path.join(self._apache_dir, 'locks')
self._docs_dir = os.path.join(self._apache_dir, 'docs')
self._conf_dir = os.path.join(self._apache_dir, 'conf')
self._conf_file = os.path.join(self._conf_dir, 'test.conf')
self._logs_dir = os.path.join(self._apache_dir, 'logs')
self._error_log = os.path.join(self._logs_dir, 'error_log')
self._tmp_dir = os.path.join(self._apache_dir, 'tmp')
self._basic_passwords = os.path.join(self._conf_dir, 'basic.passwords')
self._digest_passwords = os.path.join(self._conf_dir, 'digest.passwords')
self._apache_dir = os.path.join(env.gen_dir, "apache")
self._run_dir = os.path.join(self._apache_dir, "run")
self._lock_dir = os.path.join(self._apache_dir, "locks")
self._docs_dir = os.path.join(self._apache_dir, "docs")
self._conf_dir = os.path.join(self._apache_dir, "conf")
self._conf_file = os.path.join(self._conf_dir, "test.conf")
self._logs_dir = os.path.join(self._apache_dir, "logs")
self._error_log = os.path.join(self._logs_dir, "error_log")
self._tmp_dir = os.path.join(self._apache_dir, "tmp")
self._basic_passwords = os.path.join(self._conf_dir, "basic.passwords")
self._digest_passwords = os.path.join(self._conf_dir, "digest.passwords")
self._mods_dir = None
self._auth_digest = True
self._proxy_auth_basic = proxy_auth
self._extra_configs = {}
self._loaded_extra_configs = None
assert env.apxs
p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'],
capture_output=True, text=True)
p = subprocess.run(
args=[env.apxs, "-q", "libexecdir"], capture_output=True, text=True
)
if p.returncode != 0:
raise Exception(f'{env.apxs} failed to query libexecdir: {p}')
raise Exception(f"{env.apxs} failed to query libexecdir: {p}")
self._mods_dir = p.stdout.strip()
if self._mods_dir is None:
raise Exception('apache modules dir cannot be found')
raise Exception("apache modules dir cannot be found")
if not os.path.exists(self._mods_dir):
raise Exception(f'apache modules dir does not exist: {self._mods_dir}')
raise Exception(f"apache modules dir does not exist: {self._mods_dir}")
self._process = None
self._rmf(self._error_log)
self._init_curltest()
@ -115,50 +133,62 @@ class Httpd:
def set_proxy_auth(self, active: bool):
self._proxy_auth_basic = active
def _run(self, args, intext=''):
def _run(self, args, intext=""):
env = os.environ.copy()
env['APACHE_RUN_DIR'] = self._run_dir
env['APACHE_RUN_USER'] = os.environ['USER']
env['APACHE_LOCK_DIR'] = self._lock_dir
env['APACHE_CONFDIR'] = self._apache_dir
p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
cwd=self.env.gen_dir,
input=intext.encode() if intext else None,
env=env)
env["APACHE_RUN_DIR"] = self._run_dir
env["APACHE_RUN_USER"] = os.environ["USER"]
env["APACHE_LOCK_DIR"] = self._lock_dir
env["APACHE_CONFDIR"] = self._apache_dir
p = subprocess.run(
args,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=self.env.gen_dir,
input=intext.encode() if intext else None,
env=env,
)
start = datetime.now()
return ExecResult(args=args, exit_code=p.returncode,
stdout=p.stdout.decode().splitlines(),
stderr=p.stderr.decode().splitlines(),
duration=datetime.now() - start)
return ExecResult(
args=args,
exit_code=p.returncode,
stdout=p.stdout.decode().splitlines(),
stderr=p.stderr.decode().splitlines(),
duration=datetime.now() - start,
)
def _cmd_httpd(self, cmd: str):
args = [self.env.httpd,
"-d", self._apache_dir,
"-f", self._conf_file,
"-k", cmd]
args = [
self.env.httpd,
"-d",
self._apache_dir,
"-f",
self._conf_file,
"-k",
cmd,
]
return self._run(args=args)
def start(self):
if self._process:
self.stop()
self._write_config()
with open(self._error_log, 'a') as fd:
fd.write('start of server\n')
with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd:
fd.write('start of server\n')
r = self._cmd_httpd('start')
with open(self._error_log, "a") as fd:
fd.write("start of server\n")
with open(os.path.join(self._apache_dir, "xxx"), "a") as fd:
fd.write("start of server\n")
r = self._cmd_httpd("start")
if r.exit_code != 0:
log.error(f'failed to start httpd: {r}')
log.error(f"failed to start httpd: {r}")
return False
self._loaded_extra_configs = copy.deepcopy(self._extra_configs)
return self.wait_live(timeout=timedelta(seconds=5))
def stop(self):
r = self._cmd_httpd('stop')
r = self._cmd_httpd("stop")
self._loaded_extra_configs = None
if r.exit_code == 0:
return self.wait_dead(timeout=timedelta(seconds=5))
log.fatal(f'stopping httpd failed: {r}')
log.fatal(f"stopping httpd failed: {r}")
return r.exit_code == 0
def restart(self):
@ -170,7 +200,7 @@ class Httpd:
r = self._cmd_httpd("graceful")
self._loaded_extra_configs = None
if r.exit_code != 0:
log.error(f'failed to reload httpd: {r}')
log.error(f"failed to reload httpd: {r}")
self._loaded_extra_configs = copy.deepcopy(self._extra_configs)
return self.wait_live(timeout=timedelta(seconds=5))
@ -183,22 +213,23 @@ class Httpd:
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/')
r = curl.http_get(url=f"http://{self.env.domain1}:{self.env.http_port}/")
if r.exit_code != 0:
return True
time.sleep(.1)
time.sleep(0.1)
log.debug(f"Server still responding after {timeout}")
return False
def wait_live(self, timeout: timedelta):
curl = CurlClient(env=self.env, run_dir=self._tmp_dir,
timeout=timeout.total_seconds())
curl = CurlClient(
env=self.env, run_dir=self._tmp_dir, timeout=timeout.total_seconds()
)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/')
r = curl.http_get(url=f"http://{self.env.domain1}:{self.env.http_port}/")
if r.exit_code == 0:
return True
time.sleep(.1)
time.sleep(0.1)
log.debug(f"Server still not responding after {timeout}")
return False
@ -227,266 +258,306 @@ class Httpd:
self._mkpath(self._conf_dir)
self._mkpath(self._logs_dir)
self._mkpath(self._tmp_dir)
self._mkpath(os.path.join(self._docs_dir, 'two'))
with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd:
self._mkpath(os.path.join(self._docs_dir, "two"))
with open(os.path.join(self._docs_dir, "data.json"), "w") as fd:
data = {
'server': f'{domain1}',
"server": f"{domain1}",
}
fd.write(JSONEncoder().encode(data))
with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd:
with open(os.path.join(self._docs_dir, "two/data.json"), "w") as fd:
data = {
'server': f'{domain2}',
"server": f"{domain2}",
}
fd.write(JSONEncoder().encode(data))
if self._proxy_auth_basic:
with open(self._basic_passwords, 'w') as fd:
fd.write('proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n')
with open(self._basic_passwords, "w") as fd:
fd.write("proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n")
if self._auth_digest:
with open(self._digest_passwords, 'w') as fd:
fd.write('test:restricted area:57123e269fd73d71ae0656594e938e2f\n')
self._mkpath(os.path.join(self.docs_dir, 'restricted/digest'))
with open(os.path.join(self.docs_dir, 'restricted/digest/data.json'), 'w') as fd:
with open(self._digest_passwords, "w") as fd:
fd.write("test:restricted area:57123e269fd73d71ae0656594e938e2f\n")
self._mkpath(os.path.join(self.docs_dir, "restricted/digest"))
with open(
os.path.join(self.docs_dir, "restricted/digest/data.json"), "w"
) as fd:
fd.write('{"area":"digest"}\n')
with open(self._conf_file, 'w') as fd:
with open(self._conf_file, "w") as fd:
for m in self.MODULES:
if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')):
if os.path.exists(os.path.join(self._mods_dir, f"mod_{m}.so")):
fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n')
if Httpd.MOD_CURLTEST is not None:
fd.write(f'LoadModule curltest_module "{Httpd.MOD_CURLTEST}"\n')
conf = [ # base server config
conf = [ # base server config
f'ServerRoot "{self._apache_dir}"',
'DefaultRuntimeDir logs',
'PidFile httpd.pid',
f'ErrorLog {self._error_log}',
f'LogLevel {self._get_log_level()}',
'StartServers 4',
'ReadBufferSize 16000',
'H2MinWorkers 16',
'H2MaxWorkers 256',
f'Listen {self.env.http_port}',
f'Listen {self.env.https_port}',
f'Listen {self.env.proxy_port}',
f'Listen {self.env.proxys_port}',
"DefaultRuntimeDir logs",
"PidFile httpd.pid",
f"ErrorLog {self._error_log}",
f"LogLevel {self._get_log_level()}",
"StartServers 4",
"ReadBufferSize 16000",
"H2MinWorkers 16",
"H2MaxWorkers 256",
f"Listen {self.env.http_port}",
f"Listen {self.env.https_port}",
f"Listen {self.env.proxy_port}",
f"Listen {self.env.proxys_port}",
f'TypesConfig "{self._conf_dir}/mime.types',
'SSLSessionCache "shmcb:ssl_gcache_data(32000)"',
]
if 'base' in self._extra_configs:
conf.extend(self._extra_configs['base'])
conf.extend([ # plain http host for domain1
f'<VirtualHost *:{self.env.http_port}>',
f' ServerName {domain1}',
' ServerAlias localhost',
f' DocumentRoot "{self._docs_dir}"',
' Protocols h2c http/1.1',
' H2Direct on',
])
if "base" in self._extra_configs:
conf.extend(self._extra_configs["base"])
conf.extend(
[ # plain http host for domain1
f"<VirtualHost *:{self.env.http_port}>",
f" ServerName {domain1}",
" ServerAlias localhost",
f' DocumentRoot "{self._docs_dir}"',
" Protocols h2c http/1.1",
" H2Direct on",
]
)
conf.extend(self._curltest_conf(domain1))
conf.extend([
'</VirtualHost>',
'',
])
conf.extend([ # https host for domain1, h1 + h2
f'<VirtualHost *:{self.env.https_port}>',
f' ServerName {domain1}',
' ServerAlias localhost',
' Protocols h2 http/1.1',
' SSLEngine on',
f' SSLCertificateFile {creds1.cert_file}',
f' SSLCertificateKeyFile {creds1.pkey_file}',
f' DocumentRoot "{self._docs_dir}"',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
conf.extend(
[ # https host for domain1, h1 + h2
f"<VirtualHost *:{self.env.https_port}>",
f" ServerName {domain1}",
" ServerAlias localhost",
" Protocols h2 http/1.1",
" SSLEngine on",
f" SSLCertificateFile {creds1.cert_file}",
f" SSLCertificateKeyFile {creds1.pkey_file}",
f' DocumentRoot "{self._docs_dir}"',
]
)
conf.extend(self._curltest_conf(domain1))
if domain1 in self._extra_configs:
conf.extend(self._extra_configs[domain1])
conf.extend([
'</VirtualHost>',
'',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
# Alternate to domain1 with BROTLI compression
conf.extend([ # https host for domain1, h1 + h2
f'<VirtualHost *:{self.env.https_port}>',
f' ServerName {domain1brotli}',
' Protocols h2 http/1.1',
' SSLEngine on',
f' SSLCertificateFile {creds1.cert_file}',
f' SSLCertificateKeyFile {creds1.pkey_file}',
f' DocumentRoot "{self._docs_dir}"',
' SetOutputFilter BROTLI_COMPRESS',
])
conf.extend(
[ # https host for domain1, h1 + h2
f"<VirtualHost *:{self.env.https_port}>",
f" ServerName {domain1brotli}",
" Protocols h2 http/1.1",
" SSLEngine on",
f" SSLCertificateFile {creds1.cert_file}",
f" SSLCertificateKeyFile {creds1.pkey_file}",
f' DocumentRoot "{self._docs_dir}"',
" SetOutputFilter BROTLI_COMPRESS",
]
)
conf.extend(self._curltest_conf(domain1))
if domain1 in self._extra_configs:
conf.extend(self._extra_configs[domain1])
conf.extend([
'</VirtualHost>',
'',
])
conf.extend([ # plain http host for domain2
f'<VirtualHost *:{self.env.http_port}>',
f' ServerName {domain2}',
' ServerAlias localhost',
f' DocumentRoot "{self._docs_dir}"',
' Protocols h2c http/1.1',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
conf.extend(
[ # plain http host for domain2
f"<VirtualHost *:{self.env.http_port}>",
f" ServerName {domain2}",
" ServerAlias localhost",
f' DocumentRoot "{self._docs_dir}"',
" Protocols h2c http/1.1",
]
)
conf.extend(self._curltest_conf(domain2))
conf.extend([
'</VirtualHost>',
'',
])
conf.extend([ # https host for domain2, no h2
f'<VirtualHost *:{self.env.https_port}>',
f' ServerName {domain2}',
' Protocols http/1.1',
' SSLEngine on',
f' SSLCertificateFile {creds2.cert_file}',
f' SSLCertificateKeyFile {creds2.pkey_file}',
f' DocumentRoot "{self._docs_dir}/two"',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
conf.extend(
[ # https host for domain2, no h2
f"<VirtualHost *:{self.env.https_port}>",
f" ServerName {domain2}",
" Protocols http/1.1",
" SSLEngine on",
f" SSLCertificateFile {creds2.cert_file}",
f" SSLCertificateKeyFile {creds2.pkey_file}",
f' DocumentRoot "{self._docs_dir}/two"',
]
)
conf.extend(self._curltest_conf(domain2))
if domain2 in self._extra_configs:
conf.extend(self._extra_configs[domain2])
conf.extend([
'</VirtualHost>',
'',
])
conf.extend([ # https host for expired domain
f'<VirtualHost *:{self.env.https_port}>',
f' ServerName {exp_domain}',
' Protocols h2 http/1.1',
' SSLEngine on',
f' SSLCertificateFile {exp_creds.cert_file}',
f' SSLCertificateKeyFile {exp_creds.pkey_file}',
f' DocumentRoot "{self._docs_dir}/expired"',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
conf.extend(
[ # https host for expired domain
f"<VirtualHost *:{self.env.https_port}>",
f" ServerName {exp_domain}",
" Protocols h2 http/1.1",
" SSLEngine on",
f" SSLCertificateFile {exp_creds.cert_file}",
f" SSLCertificateKeyFile {exp_creds.pkey_file}",
f' DocumentRoot "{self._docs_dir}/expired"',
]
)
conf.extend(self._curltest_conf(exp_domain))
if exp_domain in self._extra_configs:
conf.extend(self._extra_configs[exp_domain])
conf.extend([
'</VirtualHost>',
'',
])
conf.extend([ # http forward proxy
f'<VirtualHost *:{self.env.proxy_port}>',
f' ServerName {proxy_domain}',
' Protocols h2c http/1.1',
' ProxyRequests On',
' H2ProxyRequests On',
' ProxyVia On',
f' AllowCONNECT {self.env.http_port} {self.env.https_port}',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
conf.extend(
[ # http forward proxy
f"<VirtualHost *:{self.env.proxy_port}>",
f" ServerName {proxy_domain}",
" Protocols h2c http/1.1",
" ProxyRequests On",
" H2ProxyRequests On",
" ProxyVia On",
f" AllowCONNECT {self.env.http_port} {self.env.https_port}",
]
)
conf.extend(self._get_proxy_conf())
conf.extend([
'</VirtualHost>',
'',
])
conf.extend([ # https forward proxy
f'<VirtualHost *:{self.env.proxys_port}>',
f' ServerName {proxy_domain}',
' Protocols h2 http/1.1',
' SSLEngine on',
f' SSLCertificateFile {proxy_creds.cert_file}',
f' SSLCertificateKeyFile {proxy_creds.pkey_file}',
' ProxyRequests On',
' H2ProxyRequests On',
' ProxyVia On',
f' AllowCONNECT {self.env.http_port} {self.env.https_port}',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
conf.extend(
[ # https forward proxy
f"<VirtualHost *:{self.env.proxys_port}>",
f" ServerName {proxy_domain}",
" Protocols h2 http/1.1",
" SSLEngine on",
f" SSLCertificateFile {proxy_creds.cert_file}",
f" SSLCertificateKeyFile {proxy_creds.pkey_file}",
" ProxyRequests On",
" H2ProxyRequests On",
" ProxyVia On",
f" AllowCONNECT {self.env.http_port} {self.env.https_port}",
]
)
conf.extend(self._get_proxy_conf())
conf.extend([
'</VirtualHost>',
'',
])
conf.extend(
[
"</VirtualHost>",
"",
]
)
fd.write("\n".join(conf))
with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd:
fd.write("\n".join([
'text/html html',
'application/json json',
''
]))
with open(os.path.join(self._conf_dir, "mime.types"), "w") as fd:
fd.write(
"\n".join(
["text/html html", "application/json json", ""]
)
)
def _get_proxy_conf(self):
if self._proxy_auth_basic:
return [
' <Proxy "*">',
' AuthType Basic',
" AuthType Basic",
' AuthName "Restricted Proxy"',
' AuthBasicProvider file',
" AuthBasicProvider file",
f' AuthUserFile "{self._basic_passwords}"',
' Require user proxy',
' </Proxy>',
" Require user proxy",
" </Proxy>",
]
else:
return [
' <Proxy "*">',
' Require ip 127.0.0.1',
' </Proxy>',
" Require ip 127.0.0.1",
" </Proxy>",
]
def _get_log_level(self):
if self.env.verbose > 3:
return 'trace2'
return "trace2"
if self.env.verbose > 2:
return 'trace1'
return "trace1"
if self.env.verbose > 1:
return 'debug'
return 'info'
return "debug"
return "info"
def _curltest_conf(self, servername) -> List[str]:
lines = []
if Httpd.MOD_CURLTEST is not None:
lines.extend([
' Redirect 302 /data.json.302 /data.json',
' Redirect 301 /curltest/echo301 /curltest/echo',
' Redirect 302 /curltest/echo302 /curltest/echo',
' Redirect 303 /curltest/echo303 /curltest/echo',
' Redirect 307 /curltest/echo307 /curltest/echo',
' <Location /curltest/sslinfo>',
' SSLOptions StdEnvVars',
' SetHandler curltest-sslinfo',
' </Location>',
' <Location /curltest/echo>',
' SetHandler curltest-echo',
' </Location>',
' <Location /curltest/put>',
' SetHandler curltest-put',
' </Location>',
' <Location /curltest/tweak>',
' SetHandler curltest-tweak',
' </Location>',
' Redirect 302 /tweak /curltest/tweak',
' <Location /curltest/1_1>',
' SetHandler curltest-1_1-required',
' </Location>',
' <Location /curltest/shutdown_unclean>',
' SetHandler curltest-tweak',
' SetEnv force-response-1.0 1',
' </Location>',
' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1',
])
lines.extend(
[
" Redirect 302 /data.json.302 /data.json",
" Redirect 301 /curltest/echo301 /curltest/echo",
" Redirect 302 /curltest/echo302 /curltest/echo",
" Redirect 303 /curltest/echo303 /curltest/echo",
" Redirect 307 /curltest/echo307 /curltest/echo",
" <Location /curltest/sslinfo>",
" SSLOptions StdEnvVars",
" SetHandler curltest-sslinfo",
" </Location>",
" <Location /curltest/echo>",
" SetHandler curltest-echo",
" </Location>",
" <Location /curltest/put>",
" SetHandler curltest-put",
" </Location>",
" <Location /curltest/tweak>",
" SetHandler curltest-tweak",
" </Location>",
" Redirect 302 /tweak /curltest/tweak",
" <Location /curltest/1_1>",
" SetHandler curltest-1_1-required",
" </Location>",
" <Location /curltest/shutdown_unclean>",
" SetHandler curltest-tweak",
" SetEnv force-response-1.0 1",
" </Location>",
' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1',
]
)
if self._auth_digest:
lines.extend([
f' <Directory {self.docs_dir}/restricted/digest>',
' AuthType Digest',
' AuthName "restricted area"',
f' AuthDigestDomain "https://{servername}"',
' AuthBasicProvider file',
f' AuthUserFile "{self._digest_passwords}"',
' Require valid-user',
' </Directory>',
])
lines.extend(
[
f" <Directory {self.docs_dir}/restricted/digest>",
" AuthType Digest",
' AuthName "restricted area"',
f' AuthDigestDomain "https://{servername}"',
" AuthBasicProvider file",
f' AuthUserFile "{self._digest_passwords}"',
" Require valid-user",
" </Directory>",
]
)
return lines
def _init_curltest(self):
if Httpd.MOD_CURLTEST is not None:
return
local_dir = os.path.dirname(inspect.getfile(Httpd))
p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'],
capture_output=True,
cwd=os.path.join(local_dir, 'mod_curltest'))
p = subprocess.run(
[self.env.apxs, "-c", "mod_curltest.c"],
capture_output=True,
cwd=os.path.join(local_dir, "mod_curltest"),
)
rv = p.returncode
if rv != 0:
log.error(f"compiling mod_curltest failed: {p.stderr}")
raise Exception(f"compiling mod_curltest failed: {p.stderr}")
Httpd.MOD_CURLTEST = os.path.join(
local_dir, 'mod_curltest/.libs/mod_curltest.so')
local_dir, "mod_curltest/.libs/mod_curltest.so"
)

186
tests/http/testenv/nghttpx.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -40,7 +40,6 @@ log = logging.getLogger(__name__)
class Nghttpx:
def __init__(self, env: Env, port: int, https_port: int, name: str):
self.env = env
self._name = name
@ -48,11 +47,11 @@ class Nghttpx:
self._https_port = https_port
self._cmd = env.nghttpx
self._run_dir = os.path.join(env.gen_dir, name)
self._pid_file = os.path.join(self._run_dir, 'nghttpx.pid')
self._conf_file = os.path.join(self._run_dir, 'nghttpx.conf')
self._error_log = os.path.join(self._run_dir, 'nghttpx.log')
self._stderr = os.path.join(self._run_dir, 'nghttpx.stderr')
self._tmp_dir = os.path.join(self._run_dir, 'tmp')
self._pid_file = os.path.join(self._run_dir, "nghttpx.pid")
self._conf_file = os.path.join(self._run_dir, "nghttpx.conf")
self._error_log = os.path.join(self._run_dir, "nghttpx.log")
self._stderr = os.path.join(self._run_dir, "nghttpx.stderr")
self._tmp_dir = os.path.join(self._run_dir, "tmp")
self._process: Optional[subprocess.Popen] = None
self._rmf(self._pid_file)
self._rmf(self._error_log)
@ -113,15 +112,17 @@ class Nghttpx:
return False
while datetime.now() < end_wait:
try:
log.debug(f'waiting for nghttpx({running.pid}) to exit.')
log.debug(f"waiting for nghttpx({running.pid}) to exit.")
running.wait(2)
log.debug(f'nghttpx({running.pid}) terminated -> {running.returncode}')
log.debug(
f"nghttpx({running.pid}) terminated -> {running.returncode}"
)
break
except subprocess.TimeoutExpired:
log.warning(f'nghttpx({running.pid}), not shut down yet.')
log.warning(f"nghttpx({running.pid}), not shut down yet.")
os.kill(running.pid, signal.SIGQUIT)
if datetime.now() >= end_wait:
log.error(f'nghttpx({running.pid}), terminate forcefully.')
log.error(f"nghttpx({running.pid}), terminate forcefully.")
os.kill(running.pid, signal.SIGKILL)
running.terminate()
running.wait(1)
@ -133,21 +134,34 @@ class Nghttpx:
try_until = datetime.now() + timeout
while datetime.now() < try_until:
if self._https_port > 0:
check_url = f'https://{self.env.domain1}:{self._https_port}/'
r = curl.http_get(url=check_url, extra_args=[
'--trace', 'curl.trace', '--trace-time',
'--connect-timeout', '1'
])
check_url = f"https://{self.env.domain1}:{self._https_port}/"
r = curl.http_get(
url=check_url,
extra_args=[
"--trace",
"curl.trace",
"--trace-time",
"--connect-timeout",
"1",
],
)
else:
check_url = f'https://{self.env.domain1}:{self._port}/'
r = curl.http_get(url=check_url, extra_args=[
'--trace', 'curl.trace', '--trace-time',
'--http3-only', '--connect-timeout', '1'
])
check_url = f"https://{self.env.domain1}:{self._port}/"
r = curl.http_get(
url=check_url,
extra_args=[
"--trace",
"curl.trace",
"--trace-time",
"--http3-only",
"--connect-timeout",
"1",
],
)
if r.exit_code != 0:
return True
log.debug(f'waiting for nghttpx to stop responding: {r}')
time.sleep(.1)
log.debug(f"waiting for nghttpx to stop responding: {r}")
time.sleep(0.1)
log.debug(f"Server still responding after {timeout}")
return False
@ -156,21 +170,34 @@ class Nghttpx:
try_until = datetime.now() + timeout
while datetime.now() < try_until:
if self._https_port > 0:
check_url = f'https://{self.env.domain1}:{self._https_port}/'
r = curl.http_get(url=check_url, extra_args=[
'--trace', 'curl.trace', '--trace-time',
'--connect-timeout', '1'
])
check_url = f"https://{self.env.domain1}:{self._https_port}/"
r = curl.http_get(
url=check_url,
extra_args=[
"--trace",
"curl.trace",
"--trace-time",
"--connect-timeout",
"1",
],
)
else:
check_url = f'https://{self.env.domain1}:{self._port}/'
r = curl.http_get(url=check_url, extra_args=[
'--http3-only', '--trace', 'curl.trace', '--trace-time',
'--connect-timeout', '1'
])
check_url = f"https://{self.env.domain1}:{self._port}/"
r = curl.http_get(
url=check_url,
extra_args=[
"--http3-only",
"--trace",
"curl.trace",
"--trace-time",
"--connect-timeout",
"1",
],
)
if r.exit_code == 0:
return True
log.debug(f'waiting for nghttpx to become responsive: {r}')
time.sleep(.1)
log.debug(f"waiting for nghttpx to become responsive: {r}")
time.sleep(0.1)
log.error(f"Server still not responding after {timeout}")
return False
@ -183,18 +210,19 @@ class Nghttpx:
return os.makedirs(path)
def _write_config(self):
with open(self._conf_file, 'w') as fd:
fd.write('# nghttpx test config')
fd.write("\n".join([
'# do we need something here?'
]))
with open(self._conf_file, "w") as fd:
fd.write("# nghttpx test config")
fd.write("\n".join(["# do we need something here?"]))
class NghttpxQuic(Nghttpx):
def __init__(self, env: Env):
super().__init__(env=env, name='nghttpx-quic', port=env.h3_port,
https_port=env.nghttpx_https_port)
super().__init__(
env=env,
name="nghttpx-quic",
port=env.h3_port,
https_port=env.nghttpx_https_port,
)
def start(self, wait_live=True):
self._mkpath(self._tmp_dir)
@ -204,25 +232,25 @@ class NghttpxQuic(Nghttpx):
assert creds # convince pytype this isn't None
args = [
self._cmd,
f'--frontend=*,{self.env.h3_port};quic',
'--frontend-quic-early-data',
f'--frontend=*,{self.env.nghttpx_https_port};tls',
f'--backend=127.0.0.1,{self.env.https_port};{self.env.domain1};sni={self.env.domain1};proto=h2;tls',
f'--backend=127.0.0.1,{self.env.http_port}',
'--log-level=INFO',
f'--pid-file={self._pid_file}',
f'--errorlog-file={self._error_log}',
f'--conf={self._conf_file}',
f'--cacert={self.env.ca.cert_file}',
f"--frontend=*,{self.env.h3_port};quic",
"--frontend-quic-early-data",
f"--frontend=*,{self.env.nghttpx_https_port};tls",
f"--backend=127.0.0.1,{self.env.https_port};{self.env.domain1};sni={self.env.domain1};proto=h2;tls",
f"--backend=127.0.0.1,{self.env.http_port}",
"--log-level=INFO",
f"--pid-file={self._pid_file}",
f"--errorlog-file={self._error_log}",
f"--conf={self._conf_file}",
f"--cacert={self.env.ca.cert_file}",
creds.pkey_file,
creds.cert_file,
'--frontend-http3-window-size=1M',
'--frontend-http3-max-window-size=10M',
'--frontend-http3-connection-window-size=10M',
'--frontend-http3-max-connection-window-size=100M',
"--frontend-http3-window-size=1M",
"--frontend-http3-max-window-size=10M",
"--frontend-http3-connection-window-size=10M",
"--frontend-http3-max-connection-window-size=100M",
# f'--frontend-quic-debug-log',
]
ngerr = open(self._stderr, 'a')
ngerr = open(self._stderr, "a")
self._process = subprocess.Popen(args=args, stderr=ngerr)
if self._process.returncode is not None:
return False
@ -230,10 +258,10 @@ class NghttpxQuic(Nghttpx):
class NghttpxFwd(Nghttpx):
def __init__(self, env: Env):
super().__init__(env=env, name='nghttpx-fwd', port=env.h2proxys_port,
https_port=0)
super().__init__(
env=env, name="nghttpx-fwd", port=env.h2proxys_port, https_port=0
)
def start(self, wait_live=True):
self._mkpath(self._tmp_dir)
@ -243,18 +271,18 @@ class NghttpxFwd(Nghttpx):
assert creds # convince pytype this isn't None
args = [
self._cmd,
'--http2-proxy',
f'--frontend=*,{self.env.h2proxys_port}',
f'--backend=127.0.0.1,{self.env.proxy_port}',
'--log-level=INFO',
f'--pid-file={self._pid_file}',
f'--errorlog-file={self._error_log}',
f'--conf={self._conf_file}',
f'--cacert={self.env.ca.cert_file}',
"--http2-proxy",
f"--frontend=*,{self.env.h2proxys_port}",
f"--backend=127.0.0.1,{self.env.proxy_port}",
"--log-level=INFO",
f"--pid-file={self._pid_file}",
f"--errorlog-file={self._error_log}",
f"--conf={self._conf_file}",
f"--cacert={self.env.ca.cert_file}",
creds.pkey_file,
creds.cert_file,
]
ngerr = open(self._stderr, 'a')
ngerr = open(self._stderr, "a")
self._process = subprocess.Popen(args=args, stderr=ngerr)
if self._process.returncode is not None:
return False
@ -264,12 +292,12 @@ class NghttpxFwd(Nghttpx):
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
check_url = f'https://{self.env.proxy_domain}:{self.env.h2proxys_port}/'
check_url = f"https://{self.env.proxy_domain}:{self.env.h2proxys_port}/"
r = curl.http_get(url=check_url)
if r.exit_code != 0:
return True
log.debug(f'waiting for nghttpx-fwd to stop responding: {r}')
time.sleep(.1)
log.debug(f"waiting for nghttpx-fwd to stop responding: {r}")
time.sleep(0.1)
log.debug(f"Server still responding after {timeout}")
return False
@ -277,13 +305,13 @@ class NghttpxFwd(Nghttpx):
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
check_url = f'https://{self.env.proxy_domain}:{self.env.h2proxys_port}/'
r = curl.http_get(url=check_url, extra_args=[
'--trace', 'curl.trace', '--trace-time'
])
check_url = f"https://{self.env.proxy_domain}:{self.env.h2proxys_port}/"
r = curl.http_get(
url=check_url, extra_args=["--trace", "curl.trace", "--trace-time"]
)
if r.exit_code == 0:
return True
log.debug(f'waiting for nghttpx-fwd to become responsive: {r}')
time.sleep(.1)
log.debug(f"waiting for nghttpx-fwd to become responsive: {r}")
time.sleep(0.1)
log.error(f"Server still not responding after {timeout}")
return False

4
tests/http/testenv/ports.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -37,7 +37,7 @@ def alloc_ports(port_specs: Dict[str, int]) -> Dict[str, int]:
for name, ptype in port_specs.items():
try:
s = socket.socket(type=ptype)
s.bind(('', 0))
s.bind(("", 0))
ports[name] = s.getsockname()[1]
socks.append(s)
except Exception as e:

98
tests/http/testenv/vsftpd.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -39,25 +39,24 @@ log = logging.getLogger(__name__)
class VsFTPD:
def __init__(self, env: Env, with_ssl=False):
self.env = env
self._cmd = env.vsftpd
self._scheme = 'ftp'
self._scheme = "ftp"
self._with_ssl = with_ssl
if self._with_ssl:
self._port = self.env.ftps_port
name = 'vsftpds'
name = "vsftpds"
else:
self._port = self.env.ftp_port
name = 'vsftpd'
name = "vsftpd"
self._vsftpd_dir = os.path.join(env.gen_dir, name)
self._run_dir = os.path.join(self._vsftpd_dir, 'run')
self._docs_dir = os.path.join(self._vsftpd_dir, 'docs')
self._tmp_dir = os.path.join(self._vsftpd_dir, 'tmp')
self._conf_file = os.path.join(self._vsftpd_dir, 'test.conf')
self._pid_file = os.path.join(self._vsftpd_dir, 'vsftpd.pid')
self._error_log = os.path.join(self._vsftpd_dir, 'vsftpd.log')
self._run_dir = os.path.join(self._vsftpd_dir, "run")
self._docs_dir = os.path.join(self._vsftpd_dir, "docs")
self._tmp_dir = os.path.join(self._vsftpd_dir, "tmp")
self._conf_file = os.path.join(self._vsftpd_dir, "test.conf")
self._pid_file = os.path.join(self._vsftpd_dir, "vsftpd.pid")
self._error_log = os.path.join(self._vsftpd_dir, "vsftpd.log")
self._process = None
self.clear_logs()
@ -116,9 +115,9 @@ class VsFTPD:
self._write_config()
args = [
self._cmd,
f'{self._conf_file}',
f"{self._conf_file}",
]
procerr = open(self._error_log, 'a')
procerr = open(self._error_log, "a")
self._process = subprocess.Popen(args=args, stderr=procerr)
if self._process.returncode is not None:
return False
@ -128,12 +127,12 @@ class VsFTPD:
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
check_url = f'{self._scheme}://{self.domain}:{self.port}/'
r = curl.ftp_get(urls=[check_url], extra_args=['-v'])
check_url = f"{self._scheme}://{self.domain}:{self.port}/"
r = curl.ftp_get(urls=[check_url], extra_args=["-v"])
if r.exit_code != 0:
return True
log.debug(f'waiting for vsftpd to stop responding: {r}')
time.sleep(.1)
log.debug(f"waiting for vsftpd to stop responding: {r}")
time.sleep(0.1)
log.debug(f"Server still responding after {timeout}")
return False
@ -141,14 +140,15 @@ class VsFTPD:
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
try_until = datetime.now() + timeout
while datetime.now() < try_until:
check_url = f'{self._scheme}://{self.domain}:{self.port}/'
r = curl.ftp_get(urls=[check_url], extra_args=[
'--trace', 'curl-start.trace', '--trace-time'
])
check_url = f"{self._scheme}://{self.domain}:{self.port}/"
r = curl.ftp_get(
urls=[check_url],
extra_args=["--trace", "curl-start.trace", "--trace-time"],
)
if r.exit_code == 0:
return True
log.debug(f'waiting for vsftpd to become responsive: {r}')
time.sleep(.1)
log.debug(f"waiting for vsftpd to become responsive: {r}")
time.sleep(0.1)
log.error(f"Server still not responding after {timeout}")
return False
@ -164,34 +164,36 @@ class VsFTPD:
self._mkpath(self._docs_dir)
self._mkpath(self._tmp_dir)
conf = [ # base server config
'listen=YES',
'run_as_launching_user=YES',
'#listen_address=127.0.0.1',
f'listen_port={self.port}',
'local_enable=NO',
'anonymous_enable=YES',
f'anon_root={self._docs_dir}',
'dirmessage_enable=YES',
'write_enable=YES',
'anon_upload_enable=YES',
'log_ftp_protocol=YES',
'xferlog_enable=YES',
'xferlog_std_format=NO',
f'vsftpd_log_file={self._error_log}',
'\n',
"listen=YES",
"run_as_launching_user=YES",
"#listen_address=127.0.0.1",
f"listen_port={self.port}",
"local_enable=NO",
"anonymous_enable=YES",
f"anon_root={self._docs_dir}",
"dirmessage_enable=YES",
"write_enable=YES",
"anon_upload_enable=YES",
"log_ftp_protocol=YES",
"xferlog_enable=YES",
"xferlog_std_format=NO",
f"vsftpd_log_file={self._error_log}",
"\n",
]
if self._with_ssl:
creds = self.env.get_credentials(self.domain)
assert creds # convince pytype this isn't None
conf.extend([
'ssl_enable=YES',
'debug_ssl=YES',
'allow_anon_ssl=YES',
f'rsa_cert_file={creds.cert_file}',
f'rsa_private_key_file={creds.pkey_file}',
# require_ssl_reuse=YES means ctrl and data connection need to use the same session
'require_ssl_reuse=NO',
])
conf.extend(
[
"ssl_enable=YES",
"debug_ssl=YES",
"allow_anon_ssl=YES",
f"rsa_cert_file={creds.cert_file}",
f"rsa_private_key_file={creds.pkey_file}",
# require_ssl_reuse=YES means ctrl and data connection need to use the same session
"require_ssl_reuse=NO",
]
)
with open(self._conf_file, 'w') as fd:
with open(self._conf_file, "w") as fd:
fd.write("\n".join(conf))

12
tests/http/testenv/ws_echo_server.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
@ -46,11 +46,13 @@ async def run_server(port):
def main():
parser = argparse.ArgumentParser(prog='scorecard', description="""
parser = argparse.ArgumentParser(
prog="scorecard",
description="""
Run a websocket echo server.
""")
parser.add_argument("--port", type=int,
default=9876, help="port to listen on")
""",
)
parser.add_argument("--port", type=int, default=9876, help="port to listen on")
args = parser.parse_args()
logging.basicConfig(