mirror of
https://github.com/curl/curl.git
synced 2026-08-26 18:23:32 +03:00
pytest-xdist: pytest in parallel
Require now pytest-xdist from tests/http/requirements.txt and run pytest in 'auto' parallel mode (counts cpu cores). For CI runs, set the worker count to 4, overriding the core count of 2 exposed in the images. - use Filelock to generate allocated ports at start for all workers and have subsequent workers just read the file and take the ports for their slot - make httpd config clearing a function fixture so every test starts with a clean httpd config - have fixture `configures_httpd` as parameter of test cases that configure httpd anyway, saving one reload - add pytest-xdist and filelock to required pyhton modules - add installs to ruff CI - give live checks waiting for a server to start up longer time - add fixtures to tests that rely on a server - do not stop servers unnecessarily. failures may not start them properly again, leading to unexpected fails in whatever follows - add a https: port to httpd that is *not* back by QUIC to allow failover tests without stopping the QUIC server Closes #17295
This commit is contained in:
Stefan Eissing
Daniel Stenberg
parent
f0bf43e209
commit
30ef79ed93
39 changed files with 644 additions and 449 deletions
|
|
@ -26,20 +26,27 @@
|
|||
#
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import timedelta, datetime
|
||||
from json import JSONEncoder
|
||||
from typing import Dict
|
||||
|
||||
from .curl import CurlClient
|
||||
from .env import Env
|
||||
|
||||
from .ports import alloc_ports_and_do
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Caddy:
|
||||
|
||||
PORT_SPECS = {
|
||||
'caddy': socket.SOCK_STREAM,
|
||||
'caddys': socket.SOCK_STREAM,
|
||||
}
|
||||
|
||||
def __init__(self, env: Env):
|
||||
self.env = env
|
||||
self._caddy = os.environ['CADDY'] if 'CADDY' in os.environ else env.caddy
|
||||
|
|
@ -49,6 +56,8 @@ class Caddy:
|
|||
self._error_log = os.path.join(self._caddy_dir, 'caddy.log')
|
||||
self._tmp_dir = os.path.join(self._caddy_dir, 'tmp')
|
||||
self._process = None
|
||||
self._http_port = 0
|
||||
self._https_port = 0
|
||||
self._rmf(self._error_log)
|
||||
|
||||
@property
|
||||
|
|
@ -57,7 +66,7 @@ class Caddy:
|
|||
|
||||
@property
|
||||
def port(self) -> int:
|
||||
return self.env.caddy_https_port
|
||||
return self._https_port
|
||||
|
||||
def clear_logs(self):
|
||||
self._rmf(self._error_log)
|
||||
|
|
@ -73,7 +82,24 @@ class Caddy:
|
|||
return self.start()
|
||||
return True
|
||||
|
||||
def initial_start(self):
|
||||
|
||||
def startup(ports: Dict[str, int]) -> bool:
|
||||
self._http_port = ports['caddy']
|
||||
self._https_port = ports['caddys']
|
||||
if self.start():
|
||||
self.env.update_ports(ports)
|
||||
return True
|
||||
self.stop()
|
||||
self._http_port = 0
|
||||
self._https_port = 0
|
||||
return False
|
||||
|
||||
return alloc_ports_and_do(Caddy.PORT_SPECS, startup,
|
||||
self.env.gen_root, max_tries=3)
|
||||
|
||||
def start(self, wait_live=True):
|
||||
assert self._http_port > 0 and self._https_port > 0
|
||||
self._mkpath(self._tmp_dir)
|
||||
if self._process:
|
||||
self.stop()
|
||||
|
|
@ -85,12 +111,7 @@ class Caddy:
|
|||
self._process = subprocess.Popen(args=args, cwd=self._caddy_dir, stderr=caddyerr)
|
||||
if self._process.returncode is not None:
|
||||
return False
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=5))
|
||||
|
||||
def stop_if_running(self):
|
||||
if self.is_running():
|
||||
return self.stop()
|
||||
return True
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
|
||||
def stop(self, wait_dead=True):
|
||||
self._mkpath(self._tmp_dir)
|
||||
|
|
@ -155,22 +176,25 @@ class Caddy:
|
|||
with open(self._conf_file, 'w') as fd:
|
||||
conf = [ # base server config
|
||||
'{',
|
||||
f' http_port {self.env.caddy_http_port}',
|
||||
f' https_port {self.env.caddy_https_port}',
|
||||
f' servers :{self.env.caddy_https_port} {{',
|
||||
f' http_port {self._http_port}',
|
||||
f' https_port {self._https_port}',
|
||||
f' servers :{self._https_port} {{',
|
||||
' protocols h3 h2 h1',
|
||||
' }',
|
||||
'}',
|
||||
f'{domain1}:{self.env.caddy_https_port} {{',
|
||||
f'{domain1}:{self._https_port} {{',
|
||||
' file_server * {',
|
||||
f' root {self._docs_dir}',
|
||||
' }',
|
||||
f' tls {creds1.cert_file} {creds1.pkey_file}',
|
||||
'}',
|
||||
f'{domain2} {{',
|
||||
f' reverse_proxy /* http://localhost:{self.env.http_port} {{',
|
||||
' }',
|
||||
f' tls {creds2.cert_file} {creds2.pkey_file}',
|
||||
'}',
|
||||
]
|
||||
if self.env.http_port > 0:
|
||||
conf.extend([
|
||||
f'{domain2} {{',
|
||||
f' reverse_proxy /* http://localhost:{self.env.http_port} {{',
|
||||
' }',
|
||||
f' tls {creds2.cert_file} {creds2.pkey_file}',
|
||||
'}',
|
||||
])
|
||||
fd.write("\n".join(conf))
|
||||
|
|
|
|||
|
|
@ -117,20 +117,22 @@ class RunTcpDump:
|
|||
self._stdoutfile = os.path.join(self._run_dir, 'tcpdump.out')
|
||||
self._stderrfile = os.path.join(self._run_dir, 'tcpdump.err')
|
||||
|
||||
def get_rsts(self, ports: List[int]|None = None) -> Optional[List[str]]:
|
||||
if self._proc:
|
||||
raise Exception('tcpdump still running')
|
||||
lines = []
|
||||
for line in open(self._stdoutfile):
|
||||
m = re.match(r'.* IP 127\.0\.0\.1\.(\d+) [<>] 127\.0\.0\.1\.(\d+):.*', line)
|
||||
if m:
|
||||
sport = int(m.group(1))
|
||||
dport = int(m.group(2))
|
||||
if ports is None or sport in ports or dport in ports:
|
||||
lines.append(line)
|
||||
return lines
|
||||
|
||||
@property
|
||||
def stats(self) -> Optional[List[str]]:
|
||||
if self._proc:
|
||||
raise Exception('tcpdump still running')
|
||||
return [line
|
||||
for line in open(self._stdoutfile)
|
||||
if re.match(r'.* IP 127\.0\.0\.1\.\d+ [<>] 127\.0\.0\.1\.\d+:.*', line)]
|
||||
|
||||
def stats_excluding(self, src_port) -> Optional[List[str]]:
|
||||
if self._proc:
|
||||
raise Exception('tcpdump still running')
|
||||
return [line
|
||||
for line in self.stats
|
||||
if not re.match(r'.* IP 127\.0\.0\.1\.' + str(src_port) + ' >.*', line)]
|
||||
return self.get_rsts()
|
||||
|
||||
@property
|
||||
def stderr(self) -> List[str]:
|
||||
|
|
|
|||
|
|
@ -29,15 +29,16 @@ import logging
|
|||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
from configparser import ConfigParser, ExtendedInterpolation
|
||||
from datetime import timedelta
|
||||
from typing import Optional
|
||||
from typing import Optional, Dict
|
||||
|
||||
import pytest
|
||||
from filelock import FileLock
|
||||
|
||||
from .certs import CertificateSpec, Credentials, TestCA
|
||||
from .ports import alloc_ports
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -59,9 +60,16 @@ CURL = os.path.join(TOP_PATH, 'src', 'curl')
|
|||
|
||||
class EnvConfig:
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, pytestconfig: Optional[pytest.Config] = None,
|
||||
testrun_uid=None,
|
||||
worker_id=None):
|
||||
self.pytestconfig = pytestconfig
|
||||
self.testrun_uid = testrun_uid
|
||||
self.worker_id = worker_id if worker_id is not None else 'master'
|
||||
self.tests_dir = TESTS_HTTPD_PATH
|
||||
self.gen_dir = os.path.join(self.tests_dir, 'gen')
|
||||
self.gen_root = self.gen_dir = os.path.join(self.tests_dir, 'gen')
|
||||
if self.worker_id != 'master':
|
||||
self.gen_dir = os.path.join(self.gen_dir, self.worker_id)
|
||||
self.project_dir = os.path.dirname(os.path.dirname(self.tests_dir))
|
||||
self.build_dir = TOP_PATH
|
||||
self.config = DEF_CONFIG
|
||||
|
|
@ -114,19 +122,8 @@ class EnvConfig:
|
|||
prot.lower() for prot in line[11:].split(' ')
|
||||
}
|
||||
|
||||
self.ports = alloc_ports(port_specs={
|
||||
'ftp': socket.SOCK_STREAM,
|
||||
'ftps': socket.SOCK_STREAM,
|
||||
'http': socket.SOCK_STREAM,
|
||||
'https': socket.SOCK_STREAM,
|
||||
'nghttpx_https': socket.SOCK_STREAM,
|
||||
'proxy': socket.SOCK_STREAM,
|
||||
'proxys': socket.SOCK_STREAM,
|
||||
'h2proxys': socket.SOCK_STREAM,
|
||||
'caddy': socket.SOCK_STREAM,
|
||||
'caddys': socket.SOCK_STREAM,
|
||||
'ws': socket.SOCK_STREAM,
|
||||
})
|
||||
self.ports = {}
|
||||
|
||||
self.httpd = self.config['httpd']['httpd']
|
||||
self.apxs = self.config['httpd']['apxs']
|
||||
if len(self.apxs) == 0:
|
||||
|
|
@ -287,9 +284,16 @@ class EnvConfig:
|
|||
def tcpdmp(self) -> Optional[str]:
|
||||
return self._tcpdump
|
||||
|
||||
def clear_locks(self):
|
||||
ca_lock = os.path.join(self.gen_root, 'ca/ca.lock')
|
||||
if os.path.exists(ca_lock):
|
||||
os.remove(ca_lock)
|
||||
|
||||
|
||||
class Env:
|
||||
|
||||
SERVER_TIMEOUT = 30 # seconds to wait for server to come up/reload
|
||||
|
||||
CONFIG = EnvConfig()
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -434,7 +438,9 @@ class Env:
|
|||
def tcpdump() -> Optional[str]:
|
||||
return Env.CONFIG.tcpdmp
|
||||
|
||||
def __init__(self, pytestconfig=None):
|
||||
def __init__(self, pytestconfig=None, env_config=None):
|
||||
if env_config:
|
||||
Env.CONFIG = env_config
|
||||
self._verbose = pytestconfig.option.verbose \
|
||||
if pytestconfig is not None else 0
|
||||
self._ca = None
|
||||
|
|
@ -442,11 +448,14 @@ class Env:
|
|||
|
||||
def issue_certs(self):
|
||||
if self._ca is None:
|
||||
ca_dir = os.path.join(self.CONFIG.gen_dir, 'ca')
|
||||
self._ca = TestCA.create_root(name=self.CONFIG.tld,
|
||||
store_dir=ca_dir,
|
||||
key_type="rsa2048")
|
||||
self._ca.issue_certs(self.CONFIG.cert_specs)
|
||||
ca_dir = os.path.join(self.CONFIG.gen_root, 'ca')
|
||||
os.makedirs(ca_dir, exist_ok=True)
|
||||
lock_file = os.path.join(ca_dir, 'ca.lock')
|
||||
with FileLock(lock_file):
|
||||
self._ca = TestCA.create_root(name=self.CONFIG.tld,
|
||||
store_dir=ca_dir,
|
||||
key_type="rsa2048")
|
||||
self._ca.issue_certs(self.CONFIG.cert_specs)
|
||||
|
||||
def setup(self):
|
||||
os.makedirs(self.gen_dir, exist_ok=True)
|
||||
|
|
@ -475,6 +484,10 @@ class Env:
|
|||
def gen_dir(self) -> str:
|
||||
return self.CONFIG.gen_dir
|
||||
|
||||
@property
|
||||
def gen_root(self) -> str:
|
||||
return self.CONFIG.gen_root
|
||||
|
||||
@property
|
||||
def project_dir(self) -> str:
|
||||
return self.CONFIG.project_dir
|
||||
|
|
@ -519,14 +532,25 @@ class Env:
|
|||
def expired_domain(self) -> str:
|
||||
return self.CONFIG.expired_domain
|
||||
|
||||
@property
|
||||
def ports(self) -> Dict[str, int]:
|
||||
return self.CONFIG.ports
|
||||
|
||||
def update_ports(self, ports: Dict[str, int]):
|
||||
self.CONFIG.ports.update(ports)
|
||||
|
||||
@property
|
||||
def http_port(self) -> int:
|
||||
return self.CONFIG.ports['http']
|
||||
return self.CONFIG.ports.get('http', 0)
|
||||
|
||||
@property
|
||||
def https_port(self) -> int:
|
||||
return self.CONFIG.ports['https']
|
||||
|
||||
@property
|
||||
def https_only_tcp_port(self) -> int:
|
||||
return self.CONFIG.ports['https-tcp-only']
|
||||
|
||||
@property
|
||||
def nghttpx_https_port(self) -> int:
|
||||
return self.CONFIG.ports['nghttpx_https']
|
||||
|
|
|
|||
|
|
@ -27,16 +27,18 @@
|
|||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
from datetime import timedelta, datetime
|
||||
from json import JSONEncoder
|
||||
import time
|
||||
from typing import List, Union, Optional
|
||||
from typing import List, Union, Optional, Dict
|
||||
import copy
|
||||
|
||||
from .curl import CurlClient, ExecResult
|
||||
from .env import Env
|
||||
|
||||
from .ports import alloc_ports_and_do
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -61,6 +63,14 @@ class Httpd:
|
|||
|
||||
MOD_CURLTEST = None
|
||||
|
||||
PORT_SPECS = {
|
||||
'http': socket.SOCK_STREAM,
|
||||
'https': socket.SOCK_STREAM,
|
||||
'https-tcp-only': socket.SOCK_STREAM,
|
||||
'proxy': socket.SOCK_STREAM,
|
||||
'proxys': socket.SOCK_STREAM,
|
||||
}
|
||||
|
||||
def __init__(self, env: Env, proxy_auth: bool = False):
|
||||
self.env = env
|
||||
self._apache_dir = os.path.join(env.gen_dir, 'apache')
|
||||
|
|
@ -79,6 +89,7 @@ class Httpd:
|
|||
self._proxy_auth_basic = proxy_auth
|
||||
self._extra_configs = {}
|
||||
self._loaded_extra_configs = None
|
||||
self._loaded_proxy_auth = None
|
||||
assert env.apxs
|
||||
p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'],
|
||||
capture_output=True, text=True)
|
||||
|
|
@ -89,7 +100,8 @@ class Httpd:
|
|||
raise Exception('apache modules dir cannot be found')
|
||||
if not os.path.exists(self._mods_dir):
|
||||
raise Exception(f'apache modules dir does not exist: {self._mods_dir}')
|
||||
self._process = None
|
||||
self._maybe_running = False
|
||||
self.ports = {}
|
||||
self._rmf(self._error_log)
|
||||
self._init_curltest()
|
||||
|
||||
|
|
@ -138,8 +150,25 @@ class Httpd:
|
|||
"-k", cmd]
|
||||
return self._run(args=args)
|
||||
|
||||
def initial_start(self):
|
||||
|
||||
def startup(ports: Dict[str, int]) -> bool:
|
||||
self.ports.update(ports)
|
||||
if self.start():
|
||||
self.env.update_ports(ports)
|
||||
return True
|
||||
self.stop()
|
||||
self.ports.clear()
|
||||
return False
|
||||
|
||||
return alloc_ports_and_do(Httpd.PORT_SPECS, startup,
|
||||
self.env.gen_root, max_tries=3)
|
||||
|
||||
def start(self):
|
||||
if self._process:
|
||||
# assure ports are allocated
|
||||
for key, _ in Httpd.PORT_SPECS.items():
|
||||
assert self.ports[key] is not None
|
||||
if self._maybe_running:
|
||||
self.stop()
|
||||
self._write_config()
|
||||
with open(self._error_log, 'a') as fd:
|
||||
|
|
@ -147,35 +176,41 @@ class Httpd:
|
|||
with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd:
|
||||
fd.write('start of server\n')
|
||||
r = self._cmd_httpd('start')
|
||||
if r.exit_code != 0:
|
||||
if r.exit_code != 0 or len(r.stderr):
|
||||
log.error(f'failed to start httpd: {r}')
|
||||
self.stop()
|
||||
return False
|
||||
self._loaded_extra_configs = copy.deepcopy(self._extra_configs)
|
||||
return self.wait_live(timeout=timedelta(seconds=5))
|
||||
self._loaded_proxy_auth = self._proxy_auth_basic
|
||||
return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
|
||||
def stop(self):
|
||||
r = self._cmd_httpd('stop')
|
||||
self._loaded_extra_configs = None
|
||||
self._loaded_proxy_auth = None
|
||||
if r.exit_code == 0:
|
||||
return self.wait_dead(timeout=timedelta(seconds=5))
|
||||
return self.wait_dead(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
log.fatal(f'stopping httpd failed: {r}')
|
||||
return r.exit_code == 0
|
||||
|
||||
def restart(self):
|
||||
self.stop()
|
||||
return self.start()
|
||||
|
||||
def reload(self):
|
||||
self._write_config()
|
||||
r = self._cmd_httpd("graceful")
|
||||
if r.exit_code != 0:
|
||||
log.error(f'failed to reload httpd: {r}')
|
||||
return False
|
||||
self._loaded_extra_configs = None
|
||||
self._loaded_proxy_auth = None
|
||||
if r.exit_code != 0:
|
||||
log.error(f'failed to reload httpd: {r}')
|
||||
self._loaded_extra_configs = copy.deepcopy(self._extra_configs)
|
||||
return self.wait_live(timeout=timedelta(seconds=5))
|
||||
self._loaded_proxy_auth = self._proxy_auth_basic
|
||||
return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
|
||||
def reload_if_config_changed(self):
|
||||
if self._loaded_extra_configs == self._extra_configs:
|
||||
if self._maybe_running and \
|
||||
self._loaded_extra_configs == self._extra_configs and \
|
||||
self._loaded_proxy_auth == self._proxy_auth_basic:
|
||||
return True
|
||||
return self.reload()
|
||||
|
||||
|
|
@ -183,8 +218,9 @@ class Httpd:
|
|||
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
|
||||
try_until = datetime.now() + timeout
|
||||
while datetime.now() < try_until:
|
||||
r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/')
|
||||
r = curl.http_get(url=f'http://{self.env.domain1}:{self.ports["http"]}/')
|
||||
if r.exit_code != 0:
|
||||
self._maybe_running = False
|
||||
return True
|
||||
time.sleep(.1)
|
||||
log.debug(f"Server still responding after {timeout}")
|
||||
|
|
@ -195,11 +231,12 @@ class Httpd:
|
|||
timeout=timeout.total_seconds())
|
||||
try_until = datetime.now() + timeout
|
||||
while datetime.now() < try_until:
|
||||
r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/')
|
||||
r = curl.http_get(url=f'http://{self.env.domain1}:{self.ports["http"]}/')
|
||||
if r.exit_code == 0:
|
||||
self._maybe_running = True
|
||||
return True
|
||||
time.sleep(.1)
|
||||
log.debug(f"Server still not responding after {timeout}")
|
||||
log.error(f"Server still not responding after {timeout}")
|
||||
return False
|
||||
|
||||
def _rmf(self, path):
|
||||
|
|
@ -225,6 +262,7 @@ class Httpd:
|
|||
proxy_creds = self.env.get_credentials(proxy_domain)
|
||||
assert proxy_creds # convince pytype this isn't None
|
||||
self._mkpath(self._conf_dir)
|
||||
self._mkpath(self._docs_dir)
|
||||
self._mkpath(self._logs_dir)
|
||||
self._mkpath(self._tmp_dir)
|
||||
self._mkpath(os.path.join(self._docs_dir, 'two'))
|
||||
|
|
@ -257,25 +295,24 @@ class Httpd:
|
|||
f'ServerRoot "{self._apache_dir}"',
|
||||
'DefaultRuntimeDir logs',
|
||||
'PidFile httpd.pid',
|
||||
f'ServerName {self.env.tld}',
|
||||
f'ErrorLog {self._error_log}',
|
||||
f'LogLevel {self._get_log_level()}',
|
||||
'StartServers 4',
|
||||
'ReadBufferSize 16000',
|
||||
'H2MinWorkers 16',
|
||||
'H2MaxWorkers 256',
|
||||
f'Listen {self.env.http_port}',
|
||||
f'Listen {self.env.https_port}',
|
||||
f'Listen {self.env.proxy_port}',
|
||||
f'Listen {self.env.proxys_port}',
|
||||
f'TypesConfig "{self._conf_dir}/mime.types',
|
||||
'SSLSessionCache "shmcb:ssl_gcache_data(32000)"',
|
||||
'AddEncoding x-gzip .gz .tgz .gzip',
|
||||
'AddHandler type-map .var',
|
||||
]
|
||||
conf.extend([f'Listen {port}' for _, port in self.ports.items()])
|
||||
|
||||
if 'base' in self._extra_configs:
|
||||
conf.extend(self._extra_configs['base'])
|
||||
conf.extend([ # plain http host for domain1
|
||||
f'<VirtualHost *:{self.env.http_port}>',
|
||||
f'<VirtualHost *:{self.ports["http"]}>',
|
||||
f' ServerName {domain1}',
|
||||
' ServerAlias localhost',
|
||||
f' DocumentRoot "{self._docs_dir}"',
|
||||
|
|
@ -288,7 +325,24 @@ class Httpd:
|
|||
'',
|
||||
])
|
||||
conf.extend([ # https host for domain1, h1 + h2
|
||||
f'<VirtualHost *:{self.env.https_port}>',
|
||||
f'<VirtualHost *:{self.ports["https"]}>',
|
||||
f' ServerName {domain1}',
|
||||
' ServerAlias localhost',
|
||||
' Protocols h2 http/1.1',
|
||||
' SSLEngine on',
|
||||
f' SSLCertificateFile {creds1.cert_file}',
|
||||
f' SSLCertificateKeyFile {creds1.pkey_file}',
|
||||
f' DocumentRoot "{self._docs_dir}"',
|
||||
])
|
||||
conf.extend(self._curltest_conf(domain1))
|
||||
if domain1 in self._extra_configs:
|
||||
conf.extend(self._extra_configs[domain1])
|
||||
conf.extend([
|
||||
'</VirtualHost>',
|
||||
'',
|
||||
])
|
||||
conf.extend([ # https host for domain1, h1 + h2, tcp only
|
||||
f'<VirtualHost *:{self.ports["https-tcp-only"]}>',
|
||||
f' ServerName {domain1}',
|
||||
' ServerAlias localhost',
|
||||
' Protocols h2 http/1.1',
|
||||
|
|
@ -306,7 +360,7 @@ class Httpd:
|
|||
])
|
||||
# Alternate to domain1 with BROTLI compression
|
||||
conf.extend([ # https host for domain1, h1 + h2
|
||||
f'<VirtualHost *:{self.env.https_port}>',
|
||||
f'<VirtualHost *:{self.ports["https"]}>',
|
||||
f' ServerName {domain1brotli}',
|
||||
' Protocols h2 http/1.1',
|
||||
' SSLEngine on',
|
||||
|
|
@ -323,7 +377,7 @@ class Httpd:
|
|||
'',
|
||||
])
|
||||
conf.extend([ # plain http host for domain2
|
||||
f'<VirtualHost *:{self.env.http_port}>',
|
||||
f'<VirtualHost *:{self.ports["http"]}>',
|
||||
f' ServerName {domain2}',
|
||||
' ServerAlias localhost',
|
||||
f' DocumentRoot "{self._docs_dir}"',
|
||||
|
|
@ -334,8 +388,9 @@ class Httpd:
|
|||
'</VirtualHost>',
|
||||
'',
|
||||
])
|
||||
self._mkpath(os.path.join(self._docs_dir, 'two'))
|
||||
conf.extend([ # https host for domain2, no h2
|
||||
f'<VirtualHost *:{self.env.https_port}>',
|
||||
f'<VirtualHost *:{self.ports["https"]}>',
|
||||
f' ServerName {domain2}',
|
||||
' Protocols http/1.1',
|
||||
' SSLEngine on',
|
||||
|
|
@ -350,8 +405,25 @@ class Httpd:
|
|||
'</VirtualHost>',
|
||||
'',
|
||||
])
|
||||
conf.extend([ # https host for domain2, no h2, tcp only
|
||||
f'<VirtualHost *:{self.ports["https-tcp-only"]}>',
|
||||
f' ServerName {domain2}',
|
||||
' Protocols http/1.1',
|
||||
' SSLEngine on',
|
||||
f' SSLCertificateFile {creds2.cert_file}',
|
||||
f' SSLCertificateKeyFile {creds2.pkey_file}',
|
||||
f' DocumentRoot "{self._docs_dir}/two"',
|
||||
])
|
||||
conf.extend(self._curltest_conf(domain2))
|
||||
if domain2 in self._extra_configs:
|
||||
conf.extend(self._extra_configs[domain2])
|
||||
conf.extend([
|
||||
'</VirtualHost>',
|
||||
'',
|
||||
])
|
||||
self._mkpath(os.path.join(self._docs_dir, 'expired'))
|
||||
conf.extend([ # https host for expired domain
|
||||
f'<VirtualHost *:{self.env.https_port}>',
|
||||
f'<VirtualHost *:{self.ports["https"]}>',
|
||||
f' ServerName {exp_domain}',
|
||||
' Protocols h2 http/1.1',
|
||||
' SSLEngine on',
|
||||
|
|
@ -367,13 +439,13 @@ class Httpd:
|
|||
'',
|
||||
])
|
||||
conf.extend([ # http forward proxy
|
||||
f'<VirtualHost *:{self.env.proxy_port}>',
|
||||
f'<VirtualHost *:{self.ports["proxy"]}>',
|
||||
f' ServerName {proxy_domain}',
|
||||
' Protocols h2c http/1.1',
|
||||
' ProxyRequests On',
|
||||
' H2ProxyRequests On',
|
||||
' ProxyVia On',
|
||||
f' AllowCONNECT {self.env.http_port} {self.env.https_port}',
|
||||
f' AllowCONNECT {self.ports["http"]} {self.ports["https"]}',
|
||||
])
|
||||
conf.extend(self._get_proxy_conf())
|
||||
conf.extend([
|
||||
|
|
@ -381,7 +453,7 @@ class Httpd:
|
|||
'',
|
||||
])
|
||||
conf.extend([ # https forward proxy
|
||||
f'<VirtualHost *:{self.env.proxys_port}>',
|
||||
f'<VirtualHost *:{self.ports["proxys"]}>',
|
||||
f' ServerName {proxy_domain}',
|
||||
' Protocols h2 http/1.1',
|
||||
' SSLEngine on',
|
||||
|
|
@ -390,7 +462,7 @@ class Httpd:
|
|||
' ProxyRequests On',
|
||||
' H2ProxyRequests On',
|
||||
' ProxyVia On',
|
||||
f' AllowCONNECT {self.env.http_port} {self.env.https_port}',
|
||||
f' AllowCONNECT {self.ports["http"]} {self.ports["https"]}',
|
||||
])
|
||||
conf.extend(self._get_proxy_conf())
|
||||
conf.extend([
|
||||
|
|
@ -486,12 +558,17 @@ class Httpd:
|
|||
if Httpd.MOD_CURLTEST is not None:
|
||||
return
|
||||
local_dir = os.path.dirname(inspect.getfile(Httpd))
|
||||
p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'],
|
||||
capture_output=True,
|
||||
cwd=os.path.join(local_dir, 'mod_curltest'))
|
||||
out_dir = os.path.join(self.env.gen_dir, 'mod_curltest')
|
||||
out_source = os.path.join(out_dir, 'mod_curltest.c')
|
||||
if not os.path.exists(out_dir):
|
||||
os.mkdir(out_dir)
|
||||
if not os.path.exists(out_source):
|
||||
shutil.copy(os.path.join(local_dir, 'mod_curltest/mod_curltest.c'), out_source)
|
||||
p = subprocess.run([
|
||||
self.env.apxs, '-c', out_source
|
||||
], capture_output=True, cwd=out_dir)
|
||||
rv = p.returncode
|
||||
if rv != 0:
|
||||
log.error(f"compiling mod_curltest failed: {p.stderr}")
|
||||
raise Exception(f"compiling mod_curltest failed: {p.stderr}")
|
||||
Httpd.MOD_CURLTEST = os.path.join(
|
||||
local_dir, 'mod_curltest/.libs/mod_curltest.so')
|
||||
Httpd.MOD_CURLTEST = os.path.join(out_dir, '.libs/mod_curltest.so')
|
||||
|
|
|
|||
|
|
@ -27,25 +27,26 @@
|
|||
import logging
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Optional
|
||||
from typing import Optional, Dict
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from .env import Env
|
||||
from .curl import CurlClient
|
||||
|
||||
from .ports import alloc_ports_and_do
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Nghttpx:
|
||||
|
||||
def __init__(self, env: Env, port: int, https_port: int, name: str):
|
||||
def __init__(self, env: Env, name: str):
|
||||
self.env = env
|
||||
self._name = name
|
||||
self._port = port
|
||||
self._https_port = https_port
|
||||
self._port = 0
|
||||
self._https_port = 0
|
||||
self._cmd = env.nghttpx
|
||||
self._run_dir = os.path.join(env.gen_dir, name)
|
||||
self._pid_file = os.path.join(self._run_dir, 'nghttpx.pid')
|
||||
|
|
@ -81,13 +82,11 @@ class Nghttpx:
|
|||
return self.start()
|
||||
return True
|
||||
|
||||
def start(self, wait_live=True):
|
||||
def initial_start(self):
|
||||
pass
|
||||
|
||||
def stop_if_running(self):
|
||||
if self.is_running():
|
||||
return self.stop()
|
||||
return True
|
||||
def start(self, wait_live=True):
|
||||
pass
|
||||
|
||||
def stop(self, wait_dead=True):
|
||||
self._mkpath(self._tmp_dir)
|
||||
|
|
@ -125,7 +124,7 @@ class Nghttpx:
|
|||
os.kill(running.pid, signal.SIGKILL)
|
||||
running.terminate()
|
||||
running.wait(1)
|
||||
return self.wait_live(timeout=timedelta(seconds=5))
|
||||
return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
return False
|
||||
|
||||
def wait_dead(self, timeout: timedelta):
|
||||
|
|
@ -169,7 +168,6 @@ class Nghttpx:
|
|||
])
|
||||
if r.exit_code == 0:
|
||||
return True
|
||||
log.debug(f'waiting for nghttpx to become responsive: {r}')
|
||||
time.sleep(.1)
|
||||
log.error(f"Server still not responding after {timeout}")
|
||||
return False
|
||||
|
|
@ -192,9 +190,27 @@ class Nghttpx:
|
|||
|
||||
class NghttpxQuic(Nghttpx):
|
||||
|
||||
PORT_SPECS = {
|
||||
'nghttpx_https': socket.SOCK_STREAM,
|
||||
}
|
||||
|
||||
def __init__(self, env: Env):
|
||||
super().__init__(env=env, name='nghttpx-quic', port=env.h3_port,
|
||||
https_port=env.nghttpx_https_port)
|
||||
super().__init__(env=env, name='nghttpx-quic')
|
||||
self._https_port = env.https_port
|
||||
|
||||
def initial_start(self):
|
||||
|
||||
def startup(ports: Dict[str, int]) -> bool:
|
||||
self._port = ports['nghttpx_https']
|
||||
if self.start():
|
||||
self.env.update_ports(ports)
|
||||
return True
|
||||
self.stop()
|
||||
self._port = 0
|
||||
return False
|
||||
|
||||
return alloc_ports_and_do(NghttpxQuic.PORT_SPECS, startup,
|
||||
self.env.gen_root, max_tries=3)
|
||||
|
||||
def start(self, wait_live=True):
|
||||
self._mkpath(self._tmp_dir)
|
||||
|
|
@ -206,7 +222,7 @@ class NghttpxQuic(Nghttpx):
|
|||
self._cmd,
|
||||
f'--frontend=*,{self.env.h3_port};quic',
|
||||
'--frontend-quic-early-data',
|
||||
f'--frontend=*,{self.env.nghttpx_https_port};tls',
|
||||
f'--frontend=*,{self._port};tls',
|
||||
f'--backend=127.0.0.1,{self.env.https_port};{self.env.domain1};sni={self.env.domain1};proto=h2;tls',
|
||||
f'--backend=127.0.0.1,{self.env.http_port}',
|
||||
'--log-level=INFO',
|
||||
|
|
@ -226,16 +242,34 @@ class NghttpxQuic(Nghttpx):
|
|||
self._process = subprocess.Popen(args=args, stderr=ngerr)
|
||||
if self._process.returncode is not None:
|
||||
return False
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=5))
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
|
||||
|
||||
class NghttpxFwd(Nghttpx):
|
||||
|
||||
PORT_SPECS = {
|
||||
'h2proxys': socket.SOCK_STREAM,
|
||||
}
|
||||
|
||||
def __init__(self, env: Env):
|
||||
super().__init__(env=env, name='nghttpx-fwd', port=env.h2proxys_port,
|
||||
https_port=0)
|
||||
super().__init__(env=env, name='nghttpx-fwd')
|
||||
|
||||
def initial_start(self):
|
||||
|
||||
def startup(ports: Dict[str, int]) -> bool:
|
||||
self._port = ports['h2proxys']
|
||||
if self.start():
|
||||
self.env.update_ports(ports)
|
||||
return True
|
||||
self.stop()
|
||||
self._port = 0
|
||||
return False
|
||||
|
||||
return alloc_ports_and_do(NghttpxFwd.PORT_SPECS, startup,
|
||||
self.env.gen_root, max_tries=3)
|
||||
|
||||
def start(self, wait_live=True):
|
||||
assert self._port > 0
|
||||
self._mkpath(self._tmp_dir)
|
||||
if self._process:
|
||||
self.stop()
|
||||
|
|
@ -244,7 +278,7 @@ class NghttpxFwd(Nghttpx):
|
|||
args = [
|
||||
self._cmd,
|
||||
'--http2-proxy',
|
||||
f'--frontend=*,{self.env.h2proxys_port}',
|
||||
f'--frontend=*,{self._port}',
|
||||
f'--backend=127.0.0.1,{self.env.proxy_port}',
|
||||
'--log-level=INFO',
|
||||
f'--pid-file={self._pid_file}',
|
||||
|
|
@ -258,13 +292,13 @@ class NghttpxFwd(Nghttpx):
|
|||
self._process = subprocess.Popen(args=args, stderr=ngerr)
|
||||
if self._process.returncode is not None:
|
||||
return False
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=5))
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
|
||||
def wait_dead(self, timeout: timedelta):
|
||||
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
|
||||
try_until = datetime.now() + timeout
|
||||
while datetime.now() < try_until:
|
||||
check_url = f'https://{self.env.proxy_domain}:{self.env.h2proxys_port}/'
|
||||
check_url = f'https://{self.env.proxy_domain}:{self._port}/'
|
||||
r = curl.http_get(url=check_url)
|
||||
if r.exit_code != 0:
|
||||
return True
|
||||
|
|
@ -277,13 +311,12 @@ class NghttpxFwd(Nghttpx):
|
|||
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
|
||||
try_until = datetime.now() + timeout
|
||||
while datetime.now() < try_until:
|
||||
check_url = f'https://{self.env.proxy_domain}:{self.env.h2proxys_port}/'
|
||||
check_url = f'https://{self.env.proxy_domain}:{self._port}/'
|
||||
r = curl.http_get(url=check_url, extra_args=[
|
||||
'--trace', 'curl.trace', '--trace-time'
|
||||
])
|
||||
if r.exit_code == 0:
|
||||
return True
|
||||
log.debug(f'waiting for nghttpx-fwd to become responsive: {r}')
|
||||
time.sleep(.1)
|
||||
log.error(f"Server still not responding after {timeout}")
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -25,15 +25,19 @@
|
|||
###########################################################################
|
||||
#
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from collections.abc import Callable
|
||||
from typing import Dict
|
||||
|
||||
from filelock import FileLock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def alloc_ports(port_specs: Dict[str, int]) -> Dict[str, int]:
|
||||
ports = {}
|
||||
def alloc_port_set(port_specs: Dict[str, int]) -> Dict[str, int]:
|
||||
socks = []
|
||||
ports = {}
|
||||
for name, ptype in port_specs.items():
|
||||
try:
|
||||
s = socket.socket(type=ptype)
|
||||
|
|
@ -45,3 +49,15 @@ def alloc_ports(port_specs: Dict[str, int]) -> Dict[str, int]:
|
|||
for s in socks:
|
||||
s.close()
|
||||
return ports
|
||||
|
||||
|
||||
def alloc_ports_and_do(port_spec: Dict[str, int],
|
||||
do_func: Callable[[Dict[str, int]], bool],
|
||||
gen_dir, max_tries=1) -> bool:
|
||||
lock_file = os.path.join(gen_dir, 'ports.lock')
|
||||
with FileLock(lock_file):
|
||||
for _ in range(max_tries):
|
||||
port_set = alloc_port_set(port_spec)
|
||||
if do_func(port_set):
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -26,14 +26,17 @@
|
|||
#
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Dict
|
||||
|
||||
from .curl import CurlClient
|
||||
from .curl import CurlClient, ExecResult
|
||||
from .env import Env
|
||||
|
||||
from .ports import alloc_ports_and_do
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -43,16 +46,23 @@ class VsFTPD:
|
|||
def __init__(self, env: Env, with_ssl=False, ssl_implicit=False):
|
||||
self.env = env
|
||||
self._cmd = env.vsftpd
|
||||
self._port = 0
|
||||
self._with_ssl = with_ssl
|
||||
self._ssl_implicit = ssl_implicit and with_ssl
|
||||
self._scheme = 'ftps' if self._ssl_implicit else 'ftp'
|
||||
if self._with_ssl:
|
||||
self._port = self.env.ftps_port
|
||||
name = 'vsftpds'
|
||||
self.name = 'vsftpds'
|
||||
self._port_skey = 'ftps'
|
||||
self._port_specs = {
|
||||
'ftps': socket.SOCK_STREAM,
|
||||
}
|
||||
else:
|
||||
self._port = self.env.ftp_port
|
||||
name = 'vsftpd'
|
||||
self._vsftpd_dir = os.path.join(env.gen_dir, name)
|
||||
self.name = 'vsftpd'
|
||||
self._port_skey = 'ftp'
|
||||
self._port_specs = {
|
||||
'ftp': socket.SOCK_STREAM,
|
||||
}
|
||||
self._vsftpd_dir = os.path.join(env.gen_dir, self.name)
|
||||
self._run_dir = os.path.join(self._vsftpd_dir, 'run')
|
||||
self._docs_dir = os.path.join(self._vsftpd_dir, 'docs')
|
||||
self._tmp_dir = os.path.join(self._vsftpd_dir, 'tmp')
|
||||
|
|
@ -92,11 +102,6 @@ class VsFTPD:
|
|||
return self.start()
|
||||
return True
|
||||
|
||||
def stop_if_running(self):
|
||||
if self.is_running():
|
||||
return self.stop()
|
||||
return True
|
||||
|
||||
def stop(self, wait_dead=True):
|
||||
self._mkpath(self._tmp_dir)
|
||||
if self._process:
|
||||
|
|
@ -110,7 +115,22 @@ class VsFTPD:
|
|||
self.stop()
|
||||
return self.start()
|
||||
|
||||
def initial_start(self):
|
||||
|
||||
def startup(ports: Dict[str, int]) -> bool:
|
||||
self._port = ports[self._port_skey]
|
||||
if self.start():
|
||||
self.env.update_ports(ports)
|
||||
return True
|
||||
self.stop()
|
||||
self._port = 0
|
||||
return False
|
||||
|
||||
return alloc_ports_and_do(self._port_specs, startup,
|
||||
self.env.gen_root, max_tries=3)
|
||||
|
||||
def start(self, wait_live=True):
|
||||
assert self._port > 0
|
||||
self._mkpath(self._tmp_dir)
|
||||
if self._process:
|
||||
self.stop()
|
||||
|
|
@ -123,7 +143,7 @@ class VsFTPD:
|
|||
self._process = subprocess.Popen(args=args, stderr=procerr)
|
||||
if self._process.returncode is not None:
|
||||
return False
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=5))
|
||||
return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
|
||||
|
||||
def wait_dead(self, timeout: timedelta):
|
||||
curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
|
||||
|
|
@ -148,7 +168,6 @@ class VsFTPD:
|
|||
])
|
||||
if r.exit_code == 0:
|
||||
return True
|
||||
log.debug(f'waiting for vsftpd to become responsive: {r}')
|
||||
time.sleep(.1)
|
||||
log.error(f"Server still not responding after {timeout}")
|
||||
return False
|
||||
|
|
@ -199,3 +218,7 @@ class VsFTPD:
|
|||
])
|
||||
with open(self._conf_file, 'w') as fd:
|
||||
fd.write("\n".join(conf))
|
||||
|
||||
def get_data_ports(self, r: ExecResult) -> List[int]:
|
||||
return [int(m.group(1)) for line in r.trace_lines if
|
||||
(m := re.match(r'.*Connected 2nd connection to .* port (\d+)', line))]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue