romenskiy2012/curl
1
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2026-04-14 23:51:42 +03:00
Code Issues Projects Releases Packages Wiki Activity

pytest: use dante-server in CI

Browse source 
- add startup check for 'danted' to avoid fails on low cpu
- rename 'sockd' to 'danted' everywhere to clarify what we use
- add proper defaults for 'danted' for debian
- install 'dante-server' in pytest ci runs

Closes #18075
This commit is contained in:
Stefan Eissing 2025-07-29 10:15:43 +02:00 • committed by Daniel Stenberg
parent cd586149d5
commit 6b70e8a838
No known key found for this signature in database
GPG key ID: 5CC908FDB71E12C2
11 changed files with 85 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/http3-linux.yml vendored
View file

@ -151,7 +151,7 @@ jobs:
nettle-dev libp11-kit-dev libtspi-dev libunistring-dev guile-2.2-dev libtasn1-bin \
libtasn1-6-dev libidn2-0-dev gawk gperf libtss2-dev dns-root-data bison gtk-doc-tools \
texinfo texlive texlive-extra-utils autopoint libev-dev \
apache2 apache2-dev libnghttp2-dev
apache2 apache2-dev libnghttp2-dev dante-server
echo 'CC=gcc-12' >> "$GITHUB_ENV"
echo 'CXX=g++-12' >> "$GITHUB_ENV"
@ -343,7 +343,7 @@ jobs:
nettle-dev libp11-kit-dev libtspi-dev libunistring-dev guile-2.2-dev libtasn1-bin \
libtasn1-6-dev libidn2-0-dev gawk gperf libtss2-dev dns-root-data bison gtk-doc-tools \
texinfo texlive texlive-extra-utils autopoint libev-dev libuv1-dev \
apache2 apache2-dev libnghttp2-dev vsftpd
apache2 apache2-dev libnghttp2-dev vsftpd dante-server
python3 -m venv ~/venv
echo 'CC=gcc-12' >> "$GITHUB_ENV"
echo 'CXX=g++-12' >> "$GITHUB_ENV"

2
.github/workflows/linux.yml vendored
View file

@ -288,7 +288,7 @@ jobs:
env:
INSTALL_PACKAGES: >-
${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') && 'stunnel4' || '' }}
${{ contains(matrix.build.install_steps, 'pytest') && 'apache2 apache2-dev libnghttp2-dev vsftpd' || '' }}
${{ contains(matrix.build.install_steps, 'pytest') && 'apache2 apache2-dev libnghttp2-dev vsftpd dante-server' || '' }}
run: |
sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list

40
configure.ac
View file

@ -372,35 +372,35 @@ fi
AC_SUBST(HTTPD)
AC_SUBST(APXS)
dnl we'd like a sockd as test server
dnl we'd like a dante as test socks server
dnl
SOCKD_ENABLED="maybe"
AC_ARG_WITH(test-sockd, [AS_HELP_STRING([--with-test-sockd=PATH],
[where to find dante sockd for testing])],
[request_sockd=$withval], [request_sockd=check])
if test x"$request_sockd" = "xcheck" -o x"$request_sockd" = "xyes"; then
if test -x "/usr/sbin/sockd"; then
DANTED_ENABLED="maybe"
AC_ARG_WITH(test-danted, [AS_HELP_STRING([--with-test-danted=PATH],
[where to find danted socks daemon for testing])],
[request_danted=$withval], [request_danted=check])
if test x"$request_danted" = "xcheck" -o x"$request_danted" = "xyes"; then
if test -x "/usr/sbin/danted"; then
# common location on distros (debian/ubuntu)
SOCKD="/usr/sbin/sockd"
DANTED="/usr/sbin/danted"
else
AC_PATH_PROG([SOCKD], [sockd])
if test "x$SOCKD" = "x"; then
AC_PATH_PROG([SOCKD], [sockd])
AC_PATH_PROG([DANTED], [danted])
if test "x$DANTED" = "x"; then
AC_PATH_PROG([DANTED], [danted])
fi
fi
elif test x"$request_sockd" != "xno"; then
SOCKD="${request_sockd}"
if test ! -x "${SOCKD}"; then
AC_MSG_NOTICE([sockd not found as ${SOCKD}, sockd tests disabled])
SOCKD_ENABLED="no"
elif test x"$request_danted" != "xno"; then
DANTED="${request_danted}"
if test ! -x "${DANTED}"; then
AC_MSG_NOTICE([danted not found as ${DANTED}, danted tests disabled])
DANTED_ENABLED="no"
else
AC_MSG_NOTICE([using SOCKD=$SOCKD for tests])
AC_MSG_NOTICE([using DANTED=$DANTED for tests])
fi
fi
if test x"$SOCKD_ENABLED" = "xno"; then
SOCKD=""
if test x"$DANTED_ENABLED" = "xno"; then
DANTED=""
fi
AC_SUBST(SOCKD)
AC_SUBST(DANTED)
dnl the nghttpx we might use in httpd testing
if test "x$TEST_NGHTTPX" != "x" -a "x$TEST_NGHTTPX" != "xnghttpx"; then

2
docs/INSTALL-CMAKE.md
View file

@ -460,7 +460,7 @@ Details via CMake
- `CADDY`: Default: `caddy`
- `HTTPD_NGHTTPX`: Default: `nghttpx`
- `HTTPD`: Default: `apache2`
- `SOCKD`: Default: `sockd`
- `DANTED`: Default: `danted`
- `TEST_NGHTTPX`: Default: `nghttpx`
- `VSFTPD`: Default: `vsftps`

4
docs/internals/SCORECARD.md
View file

@ -54,8 +54,8 @@ Similar options are available for uploads and requests scenarios.
## sockd
If you have configured curl with `--with-test-sockd=<sockd-path>` for a
`dante sockd` server installed on your system, you can provide the scorecard
If you have configured curl with `--with-test-danted=<danted-path>` for a
`dante-server` installed on your system, you can provide the scorecard
with arguments `--socks4` or `--socks5` to test performance with a SOCKS proxy
involved. (Note: this does not work for HTTP/3)

2
docs/tests/HTTP.md
View file

@ -52,7 +52,7 @@ Via curl's `configure` script you may specify:
* `--with-test-httpd=<httpd-install-path>` if you have an Apache httpd installed somewhere else. On Debian/Ubuntu it will otherwise look into `/usr/bin` and `/usr/sbin` to find those.
* `--with-test-caddy=<caddy-install-path>` if you have a Caddy web server installed somewhere else.
* `--with-test-vsftpd=<vsftpd-install-path>` if you have a vsftpd ftp server installed somewhere else.
* `--with-test-sockd=<dante-sockd-path>` if you have `dante sockd` server installed
* `--with-test-danted=<danted-path>` if you have `dante-server` installed
## Usage Tips

10
tests/http/CMakeLists.txt
View file

@ -52,11 +52,11 @@ if(NOT HTTPD_NGHTTPX)
endif()
mark_as_advanced(HTTPD_NGHTTPX)
find_program(SOCKD "sockd")
if(NOT SOCKD)
set(SOCKD "")
find_program(DANTED "danted")
if(NOT DANTED)
set(DANTED "")
endif()
mark_as_advanced(SOCKD)
mark_as_advanced(DANTED)
# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, SOCKD, VSFTPD
# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD
configure_file("config.ini.in" "${CMAKE_CURRENT_BINARY_DIR}/config.ini" @ONLY)

4
tests/http/config.ini.in
View file

@ -38,5 +38,5 @@ caddy = @CADDY@
[vsftpd]
vsftpd = @VSFTPD@
[sockd]
sockd = @SOCKD@
[danted]
danted = @DANTED@

28
tests/http/test_40_socks.py
View file

@ -35,15 +35,15 @@ from testenv import Env, CurlClient, Dante
log = logging.getLogger(__name__)
@pytest.mark.skipif(condition=not Env.has_sockd(), reason="missing sockd")
@pytest.mark.skipif(condition=not Env.has_danted(), reason="missing danted")
class TestSocks:
@pytest.fixture(scope='class')
def sockd(self, env: Env) -> Generator[Dante, None, None]:
sockd = Dante(env=env)
assert sockd.initial_start()
yield sockd
sockd.stop()
def danted(self, env: Env) -> Generator[Dante, None, None]:
danted = Dante(env=env)
assert danted.initial_start()
yield danted
danted.stop()
@pytest.fixture(autouse=True, scope='class')
def _class_scope(self, env, httpd):
@ -52,9 +52,9 @@ class TestSocks:
env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024)
@pytest.mark.parametrize("sproto", ['socks4', 'socks5'])
def test_40_01_socks_http(self, env: Env, sproto, sockd: Dante, httpd):
def test_40_01_socks_http(self, env: Env, sproto, danted: Dante, httpd):
curl = CurlClient(env=env, socks_args=[
f'--{sproto}', f'127.0.0.1:{sockd.port}'
f'--{sproto}', f'127.0.0.1:{danted.port}'
])
url = f'http://{env.domain1}:{env.http_port}/data.json'
r = curl.http_get(url=url)
@ -62,11 +62,11 @@ class TestSocks:
@pytest.mark.parametrize("sproto", ['socks4', 'socks5'])
@pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3'])
def test_40_02_socks_https(self, env: Env, sproto, proto, sockd: Dante, httpd):
def test_40_02_socks_https(self, env: Env, sproto, proto, danted: Dante, httpd):
if proto == 'h3' and not env.have_h3():
pytest.skip("h3 not supported")
curl = CurlClient(env=env, socks_args=[
f'--{sproto}', f'127.0.0.1:{sockd.port}'
f'--{sproto}', f'127.0.0.1:{danted.port}'
])
url = f'https://{env.authority_for(env.domain1, proto)}/data.json'
r = curl.http_get(url=url, alpn_proto=proto)
@ -77,22 +77,22 @@ class TestSocks:
@pytest.mark.parametrize("sproto", ['socks4', 'socks5'])
@pytest.mark.parametrize("proto", ['http/1.1', 'h2'])
def test_40_03_dl_serial(self, env: Env, httpd, sockd, proto, sproto):
def test_40_03_dl_serial(self, env: Env, httpd, danted, proto, sproto):
count = 3
urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]'
curl = CurlClient(env=env, socks_args=[
f'--{sproto}', f'127.0.0.1:{sockd.port}'
f'--{sproto}', f'127.0.0.1:{danted.port}'
])
r = curl.http_download(urls=[urln], alpn_proto=proto)
r.check_response(count=count, http_status=200)
@pytest.mark.parametrize("sproto", ['socks4', 'socks5'])
@pytest.mark.parametrize("proto", ['http/1.1', 'h2'])
def test_40_04_ul_serial(self, env: Env, httpd, sockd, proto, sproto):
def test_40_04_ul_serial(self, env: Env, httpd, danted, proto, sproto):
fdata = os.path.join(env.gen_dir, 'data-10m')
count = 2
curl = CurlClient(env=env, socks_args=[
f'--{sproto}', f'127.0.0.1:{sockd.port}'
f'--{sproto}', f'127.0.0.1:{danted.port}'
])
url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]'
r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto)

27
tests/http/testenv/dante.py
View file

@ -28,9 +28,12 @@ import logging
import os
import socket
import subprocess
import time
from datetime import timedelta, datetime
from typing import Dict
from . import CurlClient
from .env import Env
from .ports import alloc_ports_and_do
@ -41,12 +44,12 @@ class Dante:
def __init__(self, env: Env):
self.env = env
self._cmd = env.sockd
self._cmd = env.danted
self._port = 0
self.name = 'sockd'
self._port_skey = 'sockd'
self.name = 'danted'
self._port_skey = 'danted'
self._port_specs = {
'sockd': socket.SOCK_STREAM,
'danted': socket.SOCK_STREAM,
}
self._dante_dir = os.path.join(env.gen_dir, self.name)
self._run_dir = os.path.join(self._dante_dir, 'run')
@ -124,7 +127,21 @@ class Dante:
self._process = subprocess.Popen(args=args, stderr=procerr)
if self._process.returncode is not None:
return False
return True
return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
def wait_live(self, timeout: timedelta):
curl = CurlClient(env=self.env, run_dir=self._tmp_dir,
timeout=timeout.total_seconds(), socks_args=[
'--socks5', f'127.0.0.1:{self._port}'
])
try_until = datetime.now() + timeout
while datetime.now() < try_until:
r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/')
if r.exit_code == 0:
return True
time.sleep(.1)
log.error(f"Server still not responding after {timeout}")
return False
def _rmf(self, path):
if os.path.exists(path):

30
tests/http/testenv/env.py
View file

@ -256,28 +256,28 @@ class EnvConfig:
except Exception:
self.vsftpd = None
self.sockd = self.config['sockd']['sockd']
if self.sockd == '':
self.sockd = None
self._sockd_version = None
if self.sockd is not None:
self.danted = self.config['danted']['danted']
if self.danted == '':
self.danted = None
self._danted_version = None
if self.danted is not None:
try:
p = subprocess.run(args=[self.sockd, '-v'],
p = subprocess.run(args=[self.danted, '-v'],
capture_output=True, text=True)
assert p.returncode == 0
if p.returncode != 0:
# not a working vsftpd
self.sockd = None
self.danted = None
m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stdout)
if not m:
m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stderr)
if m:
self._sockd_version = m.group(1)
self._danted_version = m.group(1)
else:
self.sockd = None
raise Exception(f'Unable to determine sockd version from: {p.stderr}')
self.danted = None
raise Exception(f'Unable to determine danted version from: {p.stderr}')
except Exception:
self.sockd = None
self.danted = None
self._tcpdump = shutil.which('tcpdump')
@ -508,8 +508,8 @@ class Env:
return Env.CONFIG.vsftpd_version
@staticmethod
def has_sockd() -> bool:
return Env.CONFIG.sockd is not None
def has_danted() -> bool:
return Env.CONFIG.danted is not None
@staticmethod
def tcpdump() -> Optional[str]:
@ -673,8 +673,8 @@ class Env:
return self.CONFIG.ports['caddy']
@property
def sockd(self) -> str:
return self.CONFIG.sockd
def danted(self) -> str:
return self.CONFIG.danted
@property
def vsftpd(self) -> str: