romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-07-27 14:28:56 +03:00
Code Issues Projects Releases Packages Wiki Activity

lib: graceful connection shutdown

Browse source 
When libcurl discards a connection there are two phases this may go
through: "shutdown" and "closing". If a connection is aborted, the
shutdown phase is skipped and it is closed right away.

The connection filters attached to the connection implement the phases
in their `do_shutdown()` and `do_close()` callbacks. Filters carry now a
`shutdown` flags next to `connected` to keep track of the shutdown
operation.

Filters are shut down from top to bottom. If a filter is not connected,
its shutdown is skipped. Notable filters that *do* something during
shutdown are HTTP/2 and TLS. HTTP/2 sends the GOAWAY frame. TLS sends
its close notify and expects to receive a close notify from the server.

As sends and receives may EAGAIN on the network, a shutdown is often not
successful right away and needs to poll the connection's socket(s). To
facilitate this, such connections are placed on a new shutdown list
inside the connection cache.

Since managing this list requires the cooperation of a multi handle,
only the connection cache belonging to a multi handle is used. If a
connection was in another cache when being discarded, it is removed
there and added to the multi's cache. If no multi handle is available at
that time, the connection is shutdown and closed in a one-time,
best-effort attempt.

When a multi handle is destroyed, all connection still on the shutdown
list are discarded with a final shutdown attempt and close. In curl
debug builds, the environment variable `CURL_GRACEFUL_SHUTDOWN` can be
set to make this graceful with a timeout in milliseconds given by the
variable.

The shutdown list is limited to the max number of connections configured
for a multi cache. Set via CURLMOPT_MAX_TOTAL_CONNECTIONS. When the
limit is reached, the oldest connection on the shutdown list is
discarded.

- In multi_wait() and multi_waitfds(), collect all connection caches
  involved (each transfer might carry its own) into a temporary list.
  Let each connection cache on the list contribute sockets and
  POLLIN/OUT events it's connections are waiting for.

- in multi_perform() collect the connection caches the same way and let
  them peform their maintenance. This will make another non-blocking
  attempt to shutdown all connections on its shutdown list.

- for event based multis (multi->socket_cb set), add the sockets and
  their poll events via the callback. When `multi_socket()` is invoked
  for a socket not known by an active transfer, forward this to the
  multi's cache for processing. On closing a connection, remove its
  socket(s) via the callback.

TLS connection filters MUST NOT send close nofity messages in their
`do_close()` implementation. The reason is that a TLS close notify
signals a success. When a connection is aborted and skips its shutdown
phase, the server needs to see a missing close notify to detect
something has gone wrong.

A graceful shutdown of FTP's data connection is performed implicitly
before regarding the upload/download as complete and continuing on the
control connection. For FTP without TLS, there is just the socket close
happening. But with TLS, the sent/received close notify signals that the
transfer is complete and healthy. Servers like `vsftpd` verify that and
reject uploads without a TLS close notify.

- added test_19_* for shutdown related tests
- test_19_01 and test_19_02 test for TCP RST packets
  which happen without a graceful shutdown and should
  no longer appear otherwise.
- add test_19_03 for handling shutdowns by the server
- add test_19_04 for handling shutdowns by curl
- add test_19_05 for event based shutdowny by server
- add test_30_06/07 and test_31_06/07 for shutdown checks
  on FTP up- and downloads.

Closes #13976
This commit is contained in:
Stefan Eissing 2024-06-19 12:40:06 +02:00 • committed by Daniel Stenberg
parent c1845dc0e2
commit c9b95c0bb3
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
33 changed files with 1313 additions and 276 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/conftest.py
View file

@ -47,7 +47,7 @@ def pytest_report_header(config):
])
if env.has_vsftpd():
report.extend([
f' VsFTPD: {env.vsftpd_version()}, ftp:{env.ftp_port}'
f' VsFTPD: {env.vsftpd_version()}, ftp:{env.ftp_port}, ftps:{env.ftps_port}'
])
return '\n'.join(report)

4
tests/data/test1542
View file

@ -58,11 +58,11 @@ Accept: */*
== Info: Connection #0 to host %HOSTIP left intact
== Info: Connection #0 to host %HOSTIP left intact
== Info: Connection #0 to host %HOSTIP left intact
== Info: Closing connection
== Info: shutting down connection #0
== Info: Connection #1 to host %HOSTIP left intact
</file>
<stripfile>
$_ = '' if (($_ !~ /left intact/) && ($_ !~ /Closing connection/))
$_ = '' if (($_ !~ /left intact/) && ($_ !~ /(closing|shutting down) connection #\d+/))
</stripfile>
</verify>
</testcase>

8
tests/http/clients/h2-download.c
View file

@ -159,6 +159,7 @@ struct transfer {
static size_t transfer_count = 1;
static struct transfer *transfers;
static int forbid_reuse = 0;
static struct transfer *get_transfer_for_easy(CURL *easy)
{
@ -239,6 +240,8 @@ static int setup(CURL *hnd, const char *url, struct transfer *t,
curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(hnd, CURLOPT_XFERINFOFUNCTION, my_progress_cb);
curl_easy_setopt(hnd, CURLOPT_XFERINFODATA, t);
if(forbid_reuse)
curl_easy_setopt(hnd, CURLOPT_FORBID_REUSE, 1L);
/* please be verbose */
if(verbose) {
@ -288,7 +291,7 @@ int main(int argc, char *argv[])
int http_version = CURL_HTTP_VERSION_2_0;
int ch;
while((ch = getopt(argc, argv, "ahm:n:A:F:P:V:")) != -1) {
while((ch = getopt(argc, argv, "afhm:n:A:F:P:V:")) != -1) {
switch(ch) {
case 'h':
usage(NULL);
@ -296,6 +299,9 @@ int main(int argc, char *argv[])
case 'a':
abort_paused = 1;
break;
case 'f':
forbid_reuse = 1;
break;
case 'm':
max_parallel = (size_t)strtol(optarg, NULL, 10);
break;

156
tests/http/test_19_shutdown.py Normal file
View file

@ -0,0 +1,156 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
# SPDX-License-Identifier: curl
#
###########################################################################
#
import difflib
import filecmp
import logging
import os
import re
from datetime import timedelta
import pytest
from testenv import Env, CurlClient, LocalClient
log = logging.getLogger(__name__)
class TestShutdown:
@pytest.fixture(autouse=True, scope='class')
def _class_scope(self, env, httpd, nghttpx):
if env.have_h3():
nghttpx.start_if_needed()
httpd.clear_extra_configs()
httpd.reload()
@pytest.fixture(autouse=True, scope='class')
def _class_scope(self, env, httpd):
indir = httpd.docs_dir
env.make_data_file(indir=indir, fname="data-10k", fsize=10*1024)
env.make_data_file(indir=indir, fname="data-100k", fsize=100*1024)
env.make_data_file(indir=indir, fname="data-1m", fsize=1024*1024)
# check with `tcpdump` that we see curl TCP RST packets
@pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
@pytest.mark.parametrize("proto", ['http/1.1'])
def test_19_01_check_tcp_rst(self, env: Env, httpd, repeat, proto):
if env.ci_run:
pytest.skip("seems not to work in CI")
curl = CurlClient(env=env)
url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]'
r = curl.http_download(urls=[url], alpn_proto=proto, with_tcpdump=True, extra_args=[
'--parallel'
])
r.check_response(http_status=200, count=2)
assert r.tcpdump
assert len(r.tcpdump.stats) != 0, f'Expected TCP RSTs packets: {r.tcpdump.stderr}'
# check with `tcpdump` that we do NOT see TCP RST when CURL_GRACEFUL_SHUTDOWN set
@pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
@pytest.mark.parametrize("proto", ['http/1.1', 'h2'])
def test_19_02_check_shutdown(self, env: Env, httpd, repeat, proto):
if not env.curl_is_debug():
pytest.skip('only works for curl debug builds')
curl = CurlClient(env=env, run_env={
'CURL_GRACEFUL_SHUTDOWN': '2000',
'CURL_DEBUG': 'ssl'
})
url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]'
r = curl.http_download(urls=[url], alpn_proto=proto, with_tcpdump=True, extra_args=[
'--parallel'
])
r.check_response(http_status=200, count=2)
assert r.tcpdump
assert len(r.tcpdump.stats) == 0, f'Unexpected TCP RSTs packets'
# run downloads where the server closes the connection after each request
@pytest.mark.parametrize("proto", ['http/1.1'])
def test_19_03_shutdown_by_server(self, env: Env, httpd, repeat, proto):
if not env.curl_is_debug():
pytest.skip('only works for curl debug builds')
count = 10
curl = CurlClient(env=env, run_env={
'CURL_GRACEFUL_SHUTDOWN': '2000',
'CURL_DEBUG': 'ssl'
})
url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak/?'\
f'id=[0-{count-1}]&with_cl&close'
r = curl.http_download(urls=[url], alpn_proto=proto)
r.check_response(http_status=200, count=count)
shutdowns = [l for l in r.trace_lines if re.match(r'.*CCACHE\] shutdown #\d+, done=1', l)]
assert len(shutdowns) == count, f'{shutdowns}'
# run downloads with CURLOPT_FORBID_REUSE set, meaning *we* close
# the connection after each request
@pytest.mark.parametrize("proto", ['http/1.1'])
def test_19_04_shutdown_by_curl(self, env: Env, httpd, proto, repeat):
if not env.curl_is_debug():
pytest.skip('only works for curl debug builds')
count = 10
docname = 'data.json'
url = f'https://localhost:{env.https_port}/{docname}'
client = LocalClient(name='h2-download', env=env, run_env={
'CURL_GRACEFUL_SHUTDOWN': '2000',
'CURL_DEBUG': 'ssl'
})
if not client.exists():
pytest.skip(f'example client not built: {client.name}')
r = client.run(args=[
'-n', f'{count}', '-f', '-V', proto, url
])
r.check_exit_code(0)
shutdowns = [l for l in r.trace_lines if re.match(r'.*CCACHE\] shutdown #\d+, done=1', l)]
assert len(shutdowns) == count, f'{shutdowns}'
# run event-based downloads with CURLOPT_FORBID_REUSE set, meaning *we* close
# the connection after each request
@pytest.mark.parametrize("proto", ['http/1.1'])
def test_19_05_event_shutdown_by_server(self, env: Env, httpd, proto, repeat):
if not env.curl_is_debug():
pytest.skip('only works for curl debug builds')
count = 10
curl = CurlClient(env=env, run_env={
# forbid connection reuse to trigger shutdowns after transfer
'CURL_FORBID_REUSE': '1',
# make socket receives block 50% of the time to delay shutdown
'CURL_DBG_SOCK_RBLOCK': '50',
'CURL_DEBUG': 'ssl'
})
url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak/?'\
f'id=[0-{count-1}]&with_cl&'
r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
'--test-event'
])
r.check_response(http_status=200, count=count)
# check that we closed all connections
closings = [l for l in r.trace_lines if re.match(r'.*CCACHE\] closing #\d+', l)]
assert len(closings) == count, f'{closings}'
# check that all connection sockets were removed from event
removes = [l for l in r.trace_lines if re.match(r'.*socket cb: socket \d+ REMOVED', l)]
assert len(removes) == count, f'{removes}'

27
tests/http/test_30_vsftpd.py
View file

@ -136,6 +136,33 @@ class TestVsFTPD:
if os.path.exists(path):
return os.remove(path)
# check with `tcpdump` if curl causes any TCP RST packets
@pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
def test_30_06_shutdownh_download(self, env: Env, vsftpd: VsFTPD, repeat):
docname = 'data-1k'
curl = CurlClient(env=env)
count = 1
url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}?[0-{count-1}]'
r = curl.ftp_get(urls=[url], with_stats=True, with_tcpdump=True)
r.check_stats(count=count, http_status=226)
assert r.tcpdump
assert len(r.tcpdump.stats) == 0, f'Unexpected TCP RSTs packets'
# check with `tcpdump` if curl causes any TCP RST packets
@pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
def test_30_07_shutdownh_upload(self, env: Env, vsftpd: VsFTPD, repeat):
docname = 'upload-1k'
curl = CurlClient(env=env)
srcfile = os.path.join(env.gen_dir, docname)
dstfile = os.path.join(vsftpd.docs_dir, docname)
self._rmf(dstfile)
count = 1
url = f'ftp://{env.ftp_domain}:{vsftpd.port}/'
r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, with_tcpdump=True)
r.check_stats(count=count, http_status=226)
assert r.tcpdump
assert len(r.tcpdump.stats) == 0, f'Unexpected TCP RSTs packets'
def check_downloads(self, client, srcfile: str, count: int,
complete: bool = True):
for i in range(count):

29
tests/http/test_31_vsftpds.py
View file

@ -143,6 +143,35 @@ class TestVsFTPD:
if os.path.exists(path):
return os.remove(path)
# check with `tcpdump` if curl causes any TCP RST packets
@pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
def test_31_06_shutdownh_download(self, env: Env, vsftpds: VsFTPD, repeat):
docname = 'data-1k'
curl = CurlClient(env=env)
count = 1
url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
r = curl.ftp_ssl_get(urls=[url], with_stats=True, with_tcpdump=True)
r.check_stats(count=count, http_status=226)
# vsftp closes control connection without niceties,
# disregard RST packets it sent from its port to curl
assert len(r.tcpdump.stats_excluding(src_port=env.ftps_port)) == 0, f'Unexpected TCP RSTs packets'
# check with `tcpdump` if curl causes any TCP RST packets
@pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
def test_31_07_shutdownh_upload(self, env: Env, vsftpds: VsFTPD, repeat):
docname = 'upload-1k'
curl = CurlClient(env=env)
srcfile = os.path.join(env.gen_dir, docname)
dstfile = os.path.join(vsftpds.docs_dir, docname)
self._rmf(dstfile)
count = 1
url = f'ftp://{env.ftp_domain}:{vsftpds.port}/'
r = curl.ftp_ssl_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, with_tcpdump=True)
r.check_stats(count=count, http_status=226)
# vsftp closes control connection without niceties,
# disregard RST packets it sent from its port to curl
assert len(r.tcpdump.stats_excluding(src_port=env.ftps_port)) == 0, f'Unexpected TCP RSTs packets'
def check_downloads(self, client, srcfile: str, count: int,
complete: bool = True):
for i in range(count):

148
tests/http/testenv/curl.py
View file

@ -27,6 +27,10 @@
import json
import logging
import os
import sys
import time
from threading import Thread
import psutil
import re
import shutil
@ -103,6 +107,85 @@ class RunProfile:
f'stats={self.stats}]'
class RunTcpDump:
def __init__(self, env, run_dir):
self._env = env
self._run_dir = run_dir
self._proc = None
self._stdoutfile = os.path.join(self._run_dir, 'tcpdump.out')
self._stderrfile = os.path.join(self._run_dir, 'tcpdump.err')
@property
def stats(self) -> Optional[List[str]]:
if self._proc:
raise Exception('tcpdump still running')
lines = []
for l in open(self._stdoutfile).readlines():
if re.match(r'.* IP 127\.0\.0\.1\.\d+ [<>] 127\.0\.0\.1\.\d+:.*', l):
lines.append(l)
return lines
def stats_excluding(self, src_port) -> Optional[List[str]]:
if self._proc:
raise Exception('tcpdump still running')
lines = []
for l in self.stats:
if not re.match(r'.* IP 127\.0\.0\.1\.' + str(src_port) + ' >.*', l):
lines.append(l)
return lines
@property
def stderr(self) -> List[str]:
if self._proc:
raise Exception('tcpdump still running')
lines = []
return open(self._stderrfile).readlines()
def sample(self):
# not sure how to make that detection reliable for all platforms
local_if = 'lo0' if sys.platform.startswith('darwin') else 'lo'
try:
tcpdump = self._env.tcpdump()
if tcpdump is None:
raise Exception('tcpdump not available')
# look with tcpdump for TCP RST packets which indicate
# we did not shut down connections cleanly
args = []
# at least on Linux, we need root permissions to run tcpdump
if sys.platform.startswith('linux'):
args.append('sudo')
args.extend([
tcpdump, '-i', local_if, '-n', 'tcp[tcpflags] & (tcp-rst)!=0'
])
with open(self._stdoutfile, 'w') as cout:
with open(self._stderrfile, 'w') as cerr:
self._proc = subprocess.Popen(args, stdout=cout, stderr=cerr,
text=True, cwd=self._run_dir,
shell=False)
assert self._proc
assert self._proc.returncode is None
while self._proc:
try:
self._proc.wait(timeout=1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
log.error(f'Tcpdump: {e}')
def start(self):
def do_sample():
self.sample()
t = Thread(target=do_sample)
t.start()
def finish(self):
if self._proc:
time.sleep(1)
self._proc.terminate()
self._proc = None
class ExecResult:
def __init__(self, args: List[str], exit_code: int,
@ -110,13 +193,15 @@ class ExecResult:
duration: Optional[timedelta] = None,
with_stats: bool = False,
exception: Optional[str] = None,
profile: Optional[RunProfile] = None):
profile: Optional[RunProfile] = None,
tcpdump: Optional[RunTcpDump] = None):
self._args = args
self._exit_code = exit_code
self._exception = exception
self._stdout = stdout
self._stderr = stderr
self._profile = profile
self._tcpdump = tcpdump
self._duration = duration if duration is not None else timedelta()
self._response = None
self._responses = []
@ -185,6 +270,10 @@ class ExecResult:
def profile(self) -> Optional[RunProfile]:
return self._profile
@property
def tcpdump(self) -> Optional[RunTcpDump]:
return self._tcpdump
@property
def response(self) -> Optional[Dict]:
return self._response
@ -359,8 +448,11 @@ class CurlClient:
'h3': '--http3-only',
}
def __init__(self, env: Env, run_dir: Optional[str] = None,
timeout: Optional[float] = None, silent: bool = False):
def __init__(self, env: Env,
run_dir: Optional[str] = None,
timeout: Optional[float] = None,
silent: bool = False,
run_env: Optional[Dict[str, str]] = None):
self.env = env
self._timeout = timeout if timeout else env.test_timeout
self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl
@ -370,6 +462,7 @@ class CurlClient:
self._headerfile = f'{self._run_dir}/curl.headers'
self._log_path = f'{self._run_dir}/curl.log'
self._silent = silent
self._run_env = run_env
self._rmrf(self._run_dir)
self._mkpath(self._run_dir)
@ -418,18 +511,21 @@ class CurlClient:
alpn_proto: Optional[str] = None,
def_tracing: bool = True,
with_stats: bool = False,
with_profile: bool = False):
with_profile: bool = False,
with_tcpdump: bool = False):
return self._raw(url, options=extra_args,
with_stats=with_stats,
alpn_proto=alpn_proto,
def_tracing=def_tracing,
with_profile=with_profile)
with_profile=with_profile,
with_tcpdump=with_tcpdump)
def http_download(self, urls: List[str],
alpn_proto: Optional[str] = None,
with_stats: bool = True,
with_headers: bool = False,
with_profile: bool = False,
with_tcpdump: bool = False,
no_save: bool = False,
extra_args: List[str] = None):
if extra_args is None:
@ -452,13 +548,15 @@ class CurlClient:
return self._raw(urls, alpn_proto=alpn_proto, options=extra_args,
with_stats=with_stats,
with_headers=with_headers,
with_profile=with_profile)
with_profile=with_profile,
with_tcpdump=with_tcpdump)
def http_upload(self, urls: List[str], data: str,
alpn_proto: Optional[str] = None,
with_stats: bool = True,
with_headers: bool = False,
with_profile: bool = False,
with_tcpdump: bool = False,
extra_args: Optional[List[str]] = None):
if extra_args is None:
extra_args = []
@ -472,7 +570,8 @@ class CurlClient:
return self._raw(urls, alpn_proto=alpn_proto, options=extra_args,
with_stats=with_stats,
with_headers=with_headers,
with_profile=with_profile)
with_profile=with_profile,
with_tcpdump=with_tcpdump)
def http_delete(self, urls: List[str],
alpn_proto: Optional[str] = None,
@ -541,6 +640,7 @@ class CurlClient:
def ftp_get(self, urls: List[str],
with_stats: bool = True,
with_profile: bool = False,
with_tcpdump: bool = False,
no_save: bool = False,
extra_args: List[str] = None):
if extra_args is None:
@ -563,11 +663,13 @@ class CurlClient:
return self._raw(urls, options=extra_args,
with_stats=with_stats,
with_headers=False,
with_profile=with_profile)
with_profile=with_profile,
with_tcpdump=with_tcpdump)
def ftp_ssl_get(self, urls: List[str],
with_stats: bool = True,
with_profile: bool = False,
with_tcpdump: bool = False,
no_save: bool = False,
extra_args: List[str] = None):
if extra_args is None:
@ -577,11 +679,13 @@ class CurlClient:
])
return self.ftp_get(urls=urls, with_stats=with_stats,
with_profile=with_profile, no_save=no_save,
with_tcpdump=with_tcpdump,
extra_args=extra_args)
def ftp_upload(self, urls: List[str], fupload,
with_stats: bool = True,
with_profile: bool = False,
with_tcpdump: bool = False,
extra_args: List[str] = None):
if extra_args is None:
extra_args = []
@ -595,11 +699,13 @@ class CurlClient:
return self._raw(urls, options=extra_args,
with_stats=with_stats,
with_headers=False,
with_profile=with_profile)
with_profile=with_profile,
with_tcpdump=with_tcpdump)
def ftp_ssl_upload(self, urls: List[str], fupload,
with_stats: bool = True,
with_profile: bool = False,
with_tcpdump: bool = False,
extra_args: List[str] = None):
if extra_args is None:
extra_args = []
@ -608,6 +714,7 @@ class CurlClient:
])
return self.ftp_upload(urls=urls, fupload=fupload,
with_stats=with_stats, with_profile=with_profile,
with_tcpdump=with_tcpdump,
extra_args=extra_args)
def response_file(self, idx: int):
@ -625,14 +732,18 @@ class CurlClient:
my_args.extend(args)
return self._run(args=my_args, with_stats=with_stats, with_profile=with_profile)
def _run(self, args, intext='', with_stats: bool = False, with_profile: bool = True):
def _run(self, args, intext='', with_stats: bool = False,
with_profile: bool = True, with_tcpdump: bool = False):
self._rmf(self._stdoutfile)
self._rmf(self._stderrfile)
self._rmf(self._headerfile)
started_at = datetime.now()
exception = None
profile = None
tcpdump = None
started_at = datetime.now()
if with_tcpdump:
tcpdump = RunTcpDump(self.env, self._run_dir)
tcpdump.start()
try:
with open(self._stdoutfile, 'w') as cout:
with open(self._stderrfile, 'w') as cerr:
@ -641,7 +752,8 @@ class CurlClient:
if self._timeout else None
log.info(f'starting: {args}')
p = subprocess.Popen(args, stderr=cerr, stdout=cout,
cwd=self._run_dir, shell=False)
cwd=self._run_dir, shell=False,
env=self._run_env)
profile = RunProfile(p.pid, started_at, self._run_dir)
if intext is not None and False:
p.communicate(input=intext.encode(), timeout=1)
@ -663,7 +775,8 @@ class CurlClient:
p = subprocess.run(args, stderr=cerr, stdout=cout,
cwd=self._run_dir, shell=False,
input=intext.encode() if intext else None,
timeout=self._timeout)
timeout=self._timeout,
env=self._run_env)
exitcode = p.returncode
except subprocess.TimeoutExpired:
now = datetime.now()
@ -672,13 +785,15 @@ class CurlClient:
f'(configured {self._timeout}s): {args}')
exitcode = -1
exception = 'TimeoutExpired'
if tcpdump:
tcpdump.finish()
coutput = open(self._stdoutfile).readlines()
cerrput = open(self._stderrfile).readlines()
return ExecResult(args=args, exit_code=exitcode, exception=exception,
stdout=coutput, stderr=cerrput,
duration=datetime.now() - started_at,
with_stats=with_stats,
profile=profile)
profile=profile, tcpdump=tcpdump)
def _raw(self, urls, intext='', timeout=None, options=None, insecure=False,
alpn_proto: Optional[str] = None,
@ -686,13 +801,14 @@ class CurlClient:
with_stats=False,
with_headers=True,
def_tracing=True,
with_profile=False):
with_profile=False,
with_tcpdump=False):
args = self._complete_args(
urls=urls, timeout=timeout, options=options, insecure=insecure,
alpn_proto=alpn_proto, force_resolve=force_resolve,
with_headers=with_headers, def_tracing=def_tracing)
r = self._run(args, intext=intext, with_stats=with_stats,
with_profile=with_profile)
with_profile=with_profile, with_tcpdump=with_tcpdump)
if r.exit_code == 0 and with_headers:
self._parse_headerfile(self._headerfile, r=r)
if r.json:

11
tests/http/testenv/env.py
View file

@ -27,6 +27,7 @@
import logging
import os
import re
import shutil
import socket
import subprocess
import sys
@ -203,6 +204,8 @@ class EnvConfig:
except Exception as e:
self.vsftpd = None
self._tcpdump = shutil.which('tcpdump')
@property
def httpd_version(self):
if self._httpd_version is None and self.apxs is not None:
@ -264,6 +267,10 @@ class EnvConfig:
def vsftpd_version(self):
return self._vsftpd_version
@property
def tcpdmp(self) -> Optional[str]:
return self._tcpdump
class Env:
@ -383,6 +390,10 @@ class Env:
def vsftpd_version() -> str:
return Env.CONFIG.vsftpd_version
@staticmethod
def tcpdump() -> Optional[str]:
return Env.CONFIG.tcpdmp
def __init__(self, pytestconfig=None):
self._verbose = pytestconfig.option.verbose \
if pytestconfig is not None else 0

43
tests/http/testenv/mod_curltest/mod_curltest.c
View file

@ -324,10 +324,11 @@ static int curltest_tweak_handler(request_rec *r)
int i, chunks = 3, error_bucket = 1;
size_t chunk_size = sizeof(buffer);
const char *request_id = "none";
apr_time_t delay = 0, chunk_delay = 0;
apr_time_t delay = 0, chunk_delay = 0, close_delay = 0;
apr_array_header_t *args = NULL;
int http_status = 200;
apr_status_t error = APR_SUCCESS, body_error = APR_SUCCESS;
int close_conn = 0, with_cl = 0;
if(strcmp(r->handler, "curltest-tweak")) {
return DECLINED;
@ -405,6 +406,21 @@ static int curltest_tweak_handler(request_rec *r)
continue;
}
}
else if(!strcmp("close_delay", arg)) {
rv = duration_parse(&close_delay, val, "s");
if(APR_SUCCESS == rv) {
continue;
}
}
}
else if(!strcmp("close", arg)) {
/* we are asked to close the connection */
close_conn = 1;
continue;
}
else if(!strcmp("with_cl", arg)) {
with_cl = 1;
continue;
}
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "query parameter not "
"understood: '%s' in %s",
@ -417,10 +433,15 @@ static int curltest_tweak_handler(request_rec *r)
ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "error_handler: processing "
"request, %s", r->args? r->args : "(no args)");
r->status = http_status;
r->clength = -1;
r->chunked = (r->proto_num >= HTTP_VERSION(1,1));
r->clength = with_cl? (chunks * chunk_size) : -1;
r->chunked = (r->proto_num >= HTTP_VERSION(1,1)) && !with_cl;
apr_table_setn(r->headers_out, "request-id", request_id);
apr_table_unset(r->headers_out, "Content-Length");
if(r->clength >= 0) {
apr_table_set(r->headers_out, "Content-Length",
apr_ltoa(r->pool, (long)r->clength));
}
else
apr_table_unset(r->headers_out, "Content-Length");
/* Discourage content-encodings */
apr_table_unset(r->headers_out, "Content-Encoding");
apr_table_setn(r->subprocess_env, "no-brotli", "1");
@ -467,9 +488,19 @@ static int curltest_tweak_handler(request_rec *r)
"error_handler: response passed");
cleanup:
if(close_conn) {
if(close_delay) {
b = apr_bucket_flush_create(c->bucket_alloc);
APR_BRIGADE_INSERT_TAIL(bb, b);
rv = ap_pass_brigade(r->output_filters, bb);
apr_brigade_cleanup(bb);
apr_sleep(close_delay);
}
r->connection->keepalive = AP_CONN_CLOSE;
}
ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r,
"error_handler: request cleanup, r->status=%d, aborted=%d",
r->status, c->aborted);
"error_handler: request cleanup, r->status=%d, aborted=%d, "
"close=%d", r->status, c->aborted, close_conn);
if(rv == APR_SUCCESS) {
return OK;
}