#!/usr/bin/env python3 # -*- coding: utf-8 -*- #*************************************************************************** # _ _ ____ _ # Project ___| | | | _ \| | # / __| | | | |_) | | # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # # Copyright (C) Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms # are also available at https://curl.se/docs/copyright.html. # # You may opt to use, copy, modify, merge, publish, distribute and/or sell # copies of the Software, and permit persons to whom the Software is # furnished to do so, under the terms of the COPYING file. # # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY # KIND, either express or implied. # # SPDX-License-Identifier: curl # ########################################################################### # import argparse import datetime import json import logging import os import re import sys from statistics import mean from typing import Any, Dict, List, Optional from testenv import ( Caddy, CurlClient, Dante, Env, ExecResult, Httpd, NghttpxQuic, RunProfile, ) log = logging.getLogger(__name__) class ScoreCardError(Exception): pass class Card: @classmethod def fmt_ms(cls, tval): return f'{int(tval*1000)} ms' if tval >= 0 else '--' @classmethod def fmt_size(cls, val): if val >= (1024*1024*1024): return f'{val / (1024*1024*1024):0.000f}GB' if val >= (1024 * 1024): return f'{val / (1024*1024):0.000f}MB' if val >= 1024: return f'{val / 1024:0.000f}KB' return f'{val:0.000f}B' @classmethod def fmt_mbs(cls, val): if val is None or val < 0: return '--' if val >= (1024*1024): return f'{val/(1024*1024):.3g} MB/s' if val >= 1024: return f'{val / 1024:.3g} KB/s' return f'{val:.3g} B/s' @classmethod def fmt_speed(cls, val): if val is None or val < 0: return '--' if val >= (10*1024*1024): return f'{(val/(1024*1024)):.3f} MB/s' if val >= (10*1024): return f'{val/1024:.3f} KB/s' return f'{val:.3f} B/s' @classmethod def fmt_speed_result(cls, val, limit): if val is None or val < 0: return '--' pct = ((val / limit) * 100) - 100 if val >= (10*1024*1024): return f'{(val/(1024*1024)):.3f} MB/s, {pct:+.1f}%' if val >= (10*1024): return f'{val/1024:.3f} KB/s, {pct:+.1f}%' return f'{val:.3f} B/s, {pct:+.1f}%' @classmethod def fmt_reqs(cls, val): return f'{val:0.000f} r/s' if val >= 0 else '--' @classmethod def mk_mbs_cell(cls, samples, profiles, errors): val = mean(samples) if len(samples) else -1 cell = { 'val': val, 'sval': Card.fmt_mbs(val) if val >= 0 else '--', } if len(profiles): cell['stats'] = RunProfile.AverageStats(profiles) if len(errors): cell['errors'] = errors return cell @classmethod def mk_speed_cell(cls, samples, profiles, errors, limit): val = mean(samples) if len(samples) else -1 cell = { 'val': val, 'sval': Card.fmt_speed_result(val, limit) if val >= 0 else '--', } if len(profiles): cell['stats'] = RunProfile.AverageStats(profiles) if len(errors): cell['errors'] = errors return cell @classmethod def mk_reqs_cell(cls, samples, profiles, errors): val = mean(samples) if len(samples) else -1 cell = { 'val': val, 'sval': Card.fmt_reqs(val) if val >= 0 else '--', } if len(profiles): cell['stats'] = RunProfile.AverageStats(profiles) if len(errors): cell['errors'] = errors return cell @classmethod def parse_size(cls, s): m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE) if m is None: raise Exception(f'unrecognized size: {s}') size = int(m.group(1)) if not m.group(2): pass elif m.group(2).lower() == 'kb': size *= 1024 elif m.group(2).lower() == 'mb': size *= 1024 * 1024 elif m.group(2).lower() == 'gb': size *= 1024 * 1024 * 1024 return size @classmethod def print_score(cls, score): print(f'Scorecard curl, protocol {score["meta"]["protocol"]} ' f'via {score["meta"]["implementation"]}/' f'{score["meta"]["implementation_version"]}') print(f'Date: {score["meta"]["date"]}') if 'curl_V' in score["meta"]: print(f'Version: {score["meta"]["curl_V"]}') if 'curl_features' in score["meta"]: print(f'Features: {score["meta"]["curl_features"]}') if 'limit-rate' in score['meta']: print(f'--limit-rate: {score["meta"]["limit-rate"]}') print(f'Samples Size: {score["meta"]["samples"]}') if 'handshakes' in score: print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}') print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} ' f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}') for key, val in score["handshakes"].items(): print(f' {key:<17} {Card.fmt_ms(val["ipv4-connect"]):>12} ' f'{Card.fmt_ms(val["ipv4-handshake"]):>12} ' f'{Card.fmt_ms(val["ipv6-connect"]):>12} ' f'{Card.fmt_ms(val["ipv6-handshake"]):>12} ' f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}' ) for name in ['downloads', 'uploads', 'requests']: if name in score: Card.print_score_table(score[name]) @classmethod def print_score_table(cls, score): cols = score['cols'] rows = score['rows'] colw = [] statw = 13 errors = [] col_has_stats = [] for idx, col in enumerate(cols): cellw = max([len(r[idx]["sval"]) for r in rows]) colw.append(max(cellw, len(col))) col_has_stats.append(False) for row in rows: if 'stats' in row[idx]: col_has_stats[idx] = True break if 'title' in score['meta']: print(score['meta']['title']) for idx, col in enumerate(cols): if col_has_stats[idx]: print(f' {col:>{colw[idx]}} {"[cpu/rss]":<{statw}}', end='') else: print(f' {col:>{colw[idx]}}', end='') print('') for row in rows: for idx, cell in enumerate(row): print(f' {cell["sval"]:>{colw[idx]}}', end='') if col_has_stats[idx]: if 'stats' in cell: s = f'[{cell["stats"]["cpu"]:>.1f}%' \ f'/{Card.fmt_size(cell["stats"]["rss"])}]' else: s = '' print(f' {s:<{statw}}', end='') if 'errors' in cell: errors.extend(cell['errors']) print('') if len(errors): print(f'Errors: {errors}') class ScoreRunner: def __init__(self, env: Env, protocol: str, server_descr: str, server_port: int, verbose: int, curl_verbose: int, download_parallel: int = 0, upload_parallel: int = 0, server_addr: Optional[str] = None, with_flame: bool = False, socks_args: Optional[List[str]] = None, limit_rate: Optional[str] = None, suppress_cl: bool = False): self.verbose = verbose self.env = env self.protocol = protocol self.server_descr = server_descr self.server_addr = server_addr self.server_port = server_port self._silent_curl = not curl_verbose self._download_parallel = download_parallel self._upload_parallel = upload_parallel self._with_flame = with_flame self._socks_args = socks_args self._limit_rate_num = 0 self._limit_rate = limit_rate if self._limit_rate: m = re.match(r'(\d+(\.\d+)?)([gmkb])?', self._limit_rate.lower()) if not m: raise Exception(f'unrecognised limit-rate: {self._limit_rate}') self._limit_rate_num = float(m.group(1)) if m.group(3) == 'g': self._limit_rate_num *= 1024*1024*1024 elif m.group(3) == 'm': self._limit_rate_num *= 1024*1024 elif m.group(3) == 'k': self._limit_rate_num *= 1024 elif m.group(3) == 'b': pass else: raise Exception(f'unrecognised limit-rate: {self._limit_rate}') self.suppress_cl = suppress_cl def info(self, msg): if self.verbose > 0: sys.stderr.write(msg) sys.stderr.flush() def mk_curl_client(self): return CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr, with_flame=self._with_flame, socks_args=self._socks_args) def handshakes(self) -> Dict[str, Any]: props = {} sample_size = 5 self.info('TLS Handshake\n') for authority in [ 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org' ]: self.info(f' {authority}...') props[authority] = {} for ipv in ['ipv4', 'ipv6']: self.info(f'{ipv}...') c_samples = [] hs_samples = [] errors = [] for _ in range(sample_size): curl = self.mk_curl_client() args = [ '--http3-only' if self.protocol == 'h3' else '--http2', f'--{ipv}', f'https://{authority}/' ] r = curl.run_direct(args=args, with_stats=True) if r.exit_code == 0 and len(r.stats) == 1: c_samples.append(r.stats[0]['time_connect']) hs_samples.append(r.stats[0]['time_appconnect']) else: errors.append(f'exit={r.exit_code}') props[authority][f'{ipv}-connect'] = mean(c_samples) \ if len(c_samples) else -1 props[authority][f'{ipv}-handshake'] = mean(hs_samples) \ if len(hs_samples) else -1 props[authority][f'{ipv}-errors'] = errors self.info('ok.\n') return props def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): fpath = os.path.join(docs_dir, fname) data1k = 1024*'x' flen = 0 with open(fpath, 'w') as fd: while flen < fsize: fd.write(data1k) flen += len(data1k) return fpath def setup_resources(self, server_docs: str, downloads: Optional[List[int]] = None): if downloads is not None: for fsize in downloads: label = Card.fmt_size(fsize) fname = f'score{label}.data' self._make_docs_file(docs_dir=server_docs, fname=fname, fsize=fsize) self._make_docs_file(docs_dir=server_docs, fname='reqs10.data', fsize=10*1024) def _check_downloads(self, r: ExecResult, count: int): error = '' if r.exit_code != 0: error += f'exit={r.exit_code} ' if r.exit_code != 0 or len(r.stats) != count: error += f'stats={len(r.stats)}/{count} ' fails = [s for s in r.stats if s['response_code'] != 200] if len(fails) > 0: error += f'{len(fails)} failed' return error if len(error) > 0 else None def dl_single(self, url: str, nsamples: int = 1): count = 1 samples = [] errors = [] profiles = [] self.info('single...') for _ in range(nsamples): curl = self.mk_curl_client() r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True, limit_rate=self._limit_rate) err = self._check_downloads(r, count) if err: errors.append(err) elif self._limit_rate: total_speed = sum([s['speed_download'] for s in r.stats]) samples.append(total_speed / len(r.stats)) profiles.append(r.profile) else: total_size = sum([s['size_download'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) if self._limit_rate: return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num) return Card.mk_mbs_cell(samples, profiles, errors) def dl_serial(self, url: str, count: int, nsamples: int = 1): samples = [] errors = [] profiles = [] url = f'{url}?[0-{count - 1}]' self.info('serial...') for _ in range(nsamples): curl = self.mk_curl_client() r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True, limit_rate=self._limit_rate) err = self._check_downloads(r, count) if err: errors.append(err) elif self._limit_rate: total_speed = sum([s['speed_download'] for s in r.stats]) samples.append(total_speed / len(r.stats)) profiles.append(r.profile) else: total_size = sum([s['size_download'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) if self._limit_rate: return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num) return Card.mk_mbs_cell(samples, profiles, errors) def dl_parallel(self, url: str, count: int, nsamples: int = 1): samples = [] errors = [] profiles = [] max_parallel = self._download_parallel if self._download_parallel > 0 else count url = f'{url}?[0-{count - 1}]' self.info('parallel...') for _ in range(nsamples): curl = self.mk_curl_client() r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True, limit_rate=self._limit_rate, extra_args=[ '--parallel', '--parallel-max', str(max_parallel) ]) err = self._check_downloads(r, count) if err: errors.append(err) elif self._limit_rate: total_speed = sum([s['speed_download'] for s in r.stats]) samples.append(total_speed / len(r.stats)) profiles.append(r.profile) else: total_size = sum([s['size_download'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) if self._limit_rate: return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num) return Card.mk_mbs_cell(samples, profiles, errors) def downloads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]: nsamples = meta['samples'] max_parallel = self._download_parallel if self._download_parallel > 0 else count cols = ['size'] if not self._download_parallel: cols.append('single') if count > 1: cols.append(f'serial({count})') if count > 1: if max_parallel == 1: cols.append(f'serial({count})') else: cols.append(f'parallel({count}x{max_parallel})') rows = [] for fsize in fsizes: row = [{ 'val': fsize, 'sval': Card.fmt_size(fsize) }] self.info(f'{row[0]["sval"]} downloads...') url = f'https://{self.env.domain1}:{self.server_port}/score{row[0]["sval"]}.data' if 'single' in cols: row.append(self.dl_single(url=url, nsamples=nsamples)) if count > 1: if 'single' in cols: row.append(self.dl_serial(url=url, count=count, nsamples=nsamples)) row.append(self.dl_parallel(url=url, count=count, nsamples=nsamples)) rows.append(row) self.info('done.\n') if self._limit_rate: title = f'Download Speed ({self.protocol}), limit={Card.fmt_speed(self._limit_rate_num)}, from {meta["server"]}' else: title = f'Downloads ({self.protocol})from {meta["server"]}' if self._socks_args: title += f' via {self._socks_args}' return { 'meta': { 'title': title, 'count': count, 'max-parallel': max_parallel, }, 'cols': cols, 'rows': rows, } def _check_uploads(self, r: ExecResult, count: int): error = '' if r.exit_code != 0: error += f'exit={r.exit_code} ' if r.exit_code != 0 or len(r.stats) != count: error += f'stats={len(r.stats)}/{count} ' fails = [s for s in r.stats if s['response_code'] != 200] if len(fails) > 0: error += f'{len(fails)} failed' for f in fails: error += f'[{f["response_code"]}]' return error if len(error) > 0 else None def ul_single(self, url: str, fpath: str, nsamples: int = 1): samples = [] errors = [] profiles = [] self.info('single...') for _ in range(nsamples): curl = self.mk_curl_client() r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, with_headers=False, with_profile=True, suppress_cl=self.suppress_cl) err = self._check_uploads(r, 1) if err: errors.append(err) else: total_size = sum([s['size_upload'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return Card.mk_mbs_cell(samples, profiles, errors) def ul_serial(self, url: str, fpath: str, count: int, nsamples: int = 1): samples = [] errors = [] profiles = [] url = f'{url}?id=[0-{count - 1}]' self.info('serial...') for _ in range(nsamples): curl = self.mk_curl_client() r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, with_headers=False, with_profile=True, suppress_cl=self.suppress_cl) err = self._check_uploads(r, count) if err: errors.append(err) else: total_size = sum([s['size_upload'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return Card.mk_mbs_cell(samples, profiles, errors) def ul_parallel(self, url: str, fpath: str, count: int, nsamples: int = 1): samples = [] errors = [] profiles = [] max_parallel = self._download_parallel if self._download_parallel > 0 else count url = f'{url}?id=[0-{count - 1}]' self.info('parallel...') for _ in range(nsamples): curl = self.mk_curl_client() r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, with_headers=False, with_profile=True, suppress_cl=self.suppress_cl, extra_args=[ '--parallel', '--parallel-max', str(max_parallel), ]) err = self._check_uploads(r, count) if err: errors.append(err) else: total_size = sum([s['size_upload'] for s in r.stats]) samples.append(total_size / r.duration.total_seconds()) profiles.append(r.profile) return Card.mk_mbs_cell(samples, profiles, errors) def uploads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]: nsamples = meta['samples'] max_parallel = self._upload_parallel if self._upload_parallel > 0 else count cols = ['size'] run_single = not self._upload_parallel run_serial = not self._upload_parallel and count > 1 run_parallel = self._upload_parallel or count > 1 if run_single: cols.append('single') if run_serial: cols.append(f'serial({count})') if run_parallel: cols.append(f'parallel({count}x{max_parallel})') rows = [] for fsize in fsizes: row = [{ 'val': fsize, 'sval': Card.fmt_size(fsize) }] self.info(f'{row[0]["sval"]} uploads...') url = f'https://{self.env.domain1}:{self.server_port}/curltest/put' fname = f'upload{row[0]["sval"]}.data' fpath = self._make_docs_file(docs_dir=self.env.gen_dir, fname=fname, fsize=fsize) if run_single: row.append(self.ul_single(url=url, fpath=fpath, nsamples=nsamples)) if run_serial: row.append(self.ul_serial(url=url, fpath=fpath, count=count, nsamples=nsamples)) if run_parallel: row.append(self.ul_parallel(url=url, fpath=fpath, count=count, nsamples=nsamples)) rows.append(row) self.info('done.\n') title = f'Uploads to {meta["server"]}' if self._socks_args: title += f' via {self._socks_args}' return { 'meta': { 'title': title, 'count': count, 'max-parallel': max_parallel, }, 'cols': cols, 'rows': rows, } def do_requests(self, url: str, count: int, max_parallel: int = 1, nsamples: int = 1): samples = [] errors = [] profiles = [] url = f'{url}?[0-{count - 1}]' extra_args = [ '-w', '%{response_code},\\n', ] if max_parallel > 1: extra_args.extend([ '--parallel', '--parallel-max', str(max_parallel) ]) self.info(f'{max_parallel}...') for _ in range(nsamples): curl = self.mk_curl_client() r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, with_headers=False, with_profile=True, with_stats=False, extra_args=extra_args) if r.exit_code != 0: errors.append(f'exit={r.exit_code}') else: samples.append(count / r.duration.total_seconds()) non_200s = 0 for line in r.stdout.splitlines(): if not line.startswith('200,'): non_200s += 1 if non_200s > 0: errors.append(f'responses != 200: {non_200s}') profiles.append(r.profile) return Card.mk_reqs_cell(samples, profiles, errors) def requests(self, count: int, meta: Dict[str, Any]) -> Dict[str, Any]: url = f'https://{self.env.domain1}:{self.server_port}/reqs10.data' fsize = 10*1024 cols = ['size', 'total'] rows = [] mparallel = meta['request_parallels'] cols.extend([f'{mp} max' for mp in mparallel]) row = [{ 'val': fsize, 'sval': Card.fmt_size(fsize) },{ 'val': count, 'sval': f'{count}', }] self.info('requests, max parallel...') row.extend([self.do_requests(url=url, count=count, max_parallel=mp, nsamples=meta["samples"]) for mp in mparallel]) rows.append(row) self.info('done.\n') title = f'Requests in parallel to {meta["server"]}' if self._socks_args: title += f' via {self._socks_args}' return { 'meta': { 'title': title, 'count': count, }, 'cols': cols, 'rows': rows, } def score(self, handshakes: bool = True, downloads: Optional[List[int]] = None, download_count: int = 50, uploads: Optional[List[int]] = None, upload_count: int = 50, req_count=5000, request_parallels=None, nsamples: int = 1, requests: bool = True): self.info(f"scoring {self.protocol} against {self.server_descr}\n") score = { 'meta': { 'curl_version': self.env.curl_version(), 'curl_V': self.env.curl_fullname(), 'curl_features': self.env.curl_features_string(), 'os': self.env.curl_os(), 'server': self.server_descr, 'samples': nsamples, 'date': f'{datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}', } } if self._limit_rate: score['meta']['limit-rate'] = self._limit_rate if self.protocol == 'h3': score['meta']['protocol'] = 'h3' if not self.env.have_h3_curl(): raise ScoreCardError('curl does not support HTTP/3') for lib in ['ngtcp2', 'quiche', 'nghttp3']: if self.env.curl_uses_lib(lib): score['meta']['implementation'] = lib break elif self.protocol == 'h2': score['meta']['protocol'] = 'h2' if not self.env.have_h2_curl(): raise ScoreCardError('curl does not support HTTP/2') for lib in ['nghttp2']: if self.env.curl_uses_lib(lib): score['meta']['implementation'] = lib break elif self.protocol == 'h1' or self.protocol == 'http/1.1': score['meta']['protocol'] = 'http/1.1' score['meta']['implementation'] = 'native' else: raise ScoreCardError(f"unknown protocol: {self.protocol}") if 'implementation' not in score['meta']: raise ScoreCardError('did not recognized protocol lib') score['meta']['implementation_version'] = Env.curl_lib_version(score['meta']['implementation']) if handshakes: score['handshakes'] = self.handshakes() if downloads and len(downloads) > 0: score['downloads'] = self.downloads(count=download_count, fsizes=downloads, meta=score['meta']) if uploads and len(uploads) > 0: score['uploads'] = self.uploads(count=upload_count, fsizes=uploads, meta=score['meta']) if requests: if request_parallels is None: request_parallels = [1, 6, 25, 50, 100, 300] score['meta']['request_parallels'] = request_parallels score['requests'] = self.requests(count=req_count, meta=score['meta']) return score def run_score(args, protocol): if protocol not in ['http/1.1', 'h1', 'h2', 'h3']: sys.stderr.write(f'ERROR: protocol "{protocol}" not known to scorecard\n') sys.exit(1) if protocol == 'h1': protocol = 'http/1.1' handshakes = True downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] if args.download_sizes is not None: downloads = [] for x in args.download_sizes: downloads.extend([Card.parse_size(s) for s in x.split(',')]) uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] if args.upload_sizes is not None: uploads = [] for x in args.upload_sizes: uploads.extend([Card.parse_size(s) for s in x.split(',')]) requests = True request_parallels = None if args.request_parallels: request_parallels = [] for x in args.request_parallels: request_parallels.extend([int(s) for s in x.split(',')]) if args.downloads or args.uploads or args.requests or args.handshakes: handshakes = args.handshakes if not args.downloads: downloads = None if not args.uploads: uploads = None requests = args.requests test_httpd = protocol != 'h3' test_caddy = protocol == 'h3' if args.caddy or args.httpd: test_caddy = args.caddy test_httpd = args.httpd rv = 0 env = Env() env.setup() env.test_timeout = None sockd = None socks_args = None if args.socks4 and args.socks5: raise ScoreCardError('unable to run --socks4 and --socks5 together') elif args.socks4 or args.socks5: sockd = Dante(env=env) if sockd: assert sockd.initial_start() socks_args = [ '--socks4' if args.socks4 else '--socks5', f'127.0.0.1:{sockd.port}', ] httpd = None nghttpx = None caddy = None try: cards = [] if args.remote: m = re.match(r'^(.+):(\d+)$', args.remote) if m is None: raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}') test_httpd = False test_caddy = False remote_addr = m.group(1) remote_port = int(m.group(2)) card = ScoreRunner(env=env, protocol=protocol, server_descr=f'Server at {args.remote}', server_addr=remote_addr, server_port=remote_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, upload_parallel=args.upload_parallel, with_flame=args.flame, socks_args=socks_args, limit_rate=args.limit_rate, suppress_cl=args.upload_no_cl) cards.append(card) if test_httpd: httpd = Httpd(env=env) assert httpd.exists(), \ f'httpd not found: {env.httpd}' httpd.clear_logs() server_docs = httpd.docs_dir assert httpd.initial_start() if protocol == 'h3': nghttpx = NghttpxQuic(env=env) nghttpx.clear_logs() assert nghttpx.initial_start() server_descr = f'nghttpx: https:{env.h3_port} [backend httpd/{env.httpd_version()}]' server_port = env.h3_port else: server_descr = f'httpd/{env.httpd_version()}' server_port = env.https_port card = ScoreRunner(env=env, protocol=protocol, server_descr=server_descr, server_port=server_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, upload_parallel=args.upload_parallel, with_flame=args.flame, socks_args=socks_args, limit_rate=args.limit_rate) card.setup_resources(server_docs, downloads) cards.append(card) if test_caddy and env.caddy: backend = '' if uploads and httpd is None: backend = f' [backend httpd: {env.httpd_version()}]' httpd = Httpd(env=env) assert httpd.exists(), \ f'httpd not found: {env.httpd}' httpd.clear_logs() assert httpd.initial_start() caddy = Caddy(env=env) caddy.clear_logs() assert caddy.initial_start() server_descr = f'Caddy/{env.caddy_version()} {backend}' server_port = caddy.port server_docs = caddy.docs_dir card = ScoreRunner(env=env, protocol=protocol, server_descr=server_descr, server_port=server_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, upload_parallel=args.upload_parallel, with_flame=args.flame, socks_args=socks_args, limit_rate=args.limit_rate) card.setup_resources(server_docs, downloads) cards.append(card) if args.start_only: print('started servers:') for card in cards: print(f'{card.server_descr}') sys.stderr.write('press [RETURN] to finish') sys.stderr.flush() sys.stdin.readline() else: for card in cards: score = card.score(handshakes=handshakes, downloads=downloads, download_count=args.download_count, uploads=uploads, upload_count=args.upload_count, req_count=args.request_count, requests=requests, request_parallels=request_parallels, nsamples=args.samples) if args.json: print(json.JSONEncoder(indent=2).encode(score)) else: Card.print_score(score) except ScoreCardError as ex: sys.stderr.write(f"ERROR: {ex}\n") rv = 1 except KeyboardInterrupt: log.warning("aborted") rv = 1 finally: if caddy: caddy.stop() if nghttpx: nghttpx.stop(wait_dead=False) if httpd: httpd.stop() if sockd: sockd.stop() return rv def print_file(filename): if not os.path.exists(filename): sys.stderr.write(f"ERROR: file does not exist {filename}\n") return 1 with open(filename) as file: data = json.load(file) Card.print_score(data) return 0 def main(): parser = argparse.ArgumentParser(prog='scorecard', description=""" Run a range of tests to give a scorecard for an HTTP protocol 'h3' or 'h2' implementation in curl. """) parser.add_argument("-v", "--verbose", action='count', default=1, help="log more output on stderr") parser.add_argument("-j", "--json", action='store_true', default=False, help="print json instead of text") parser.add_argument("--samples", action='store', type=int, metavar='number', default=1, help="how many sample runs to make") parser.add_argument("--httpd", action='store_true', default=False, help="evaluate httpd server only") parser.add_argument("--caddy", action='store_true', default=False, help="evaluate caddy server only") parser.add_argument("--curl-verbose", action='store_true', default=False, help="run curl with `-v`") parser.add_argument("--print", type=str, default=None, metavar='filename', help="print the results from a JSON file") parser.add_argument("protocol", default=None, nargs='?', help="Name of protocol to score") parser.add_argument("--start-only", action='store_true', default=False, help="only start the servers") parser.add_argument("--remote", action='store', type=str, default=None, help="score against the remote server at :") parser.add_argument("--flame", action='store_true', default = False, help="produce a flame graph on curl") parser.add_argument("--limit-rate", action='store', type=str, default=None, help="use curl's --limit-rate") parser.add_argument("-H", "--handshakes", action='store_true', default=False, help="evaluate handshakes only") parser.add_argument("-d", "--downloads", action='store_true', default=False, help="evaluate downloads") parser.add_argument("--download-sizes", action='append', type=str, metavar='numberlist', default=None, help="evaluate download size") parser.add_argument("--download-count", action='store', type=int, metavar='number', default=50, help="perform that many downloads") parser.add_argument("--download-parallel", action='store', type=int, metavar='number', default=0, help="perform that many downloads in parallel (default all)") parser.add_argument("-u", "--uploads", action='store_true', default=False, help="evaluate uploads") parser.add_argument("--upload-sizes", action='append', type=str, metavar='numberlist', default=None, help="evaluate upload size") parser.add_argument("--upload-count", action='store', type=int, metavar='number', default=50, help="perform that many uploads") parser.add_argument("--upload-parallel", action='store', type=int, metavar='number', default=0, help="perform that many uploads in parallel (default all)") parser.add_argument("--upload-no-cl", action='store_true', default=False, help="suppress content-length on upload") parser.add_argument("-r", "--requests", action='store_true', default=False, help="evaluate requests") parser.add_argument("--request-count", action='store', type=int, metavar='number', default=5000, help="perform that many requests") parser.add_argument("--request-parallels", action='append', type=str, metavar='numberlist', default=None, help="evaluate request with these max-parallel numbers") parser.add_argument("--socks4", action='store_true', default=False, help="test with SOCKS4 proxy") parser.add_argument("--socks5", action='store_true', default=False, help="test with SOCKS5 proxy") args = parser.parse_args() if args.verbose > 0: console = logging.StreamHandler() console.setLevel(logging.INFO) console.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) logging.getLogger('').addHandler(console) if args.print: rv = print_file(args.print) elif not args.protocol: parser.print_usage() rv = 1 else: rv = run_score(args, args.protocol) sys.exit(rv) if __name__ == "__main__": main()