"""Compare the generated file with pinned standalone Uvicorn (one worker each)."""
import argparse
import http.client
import json
from pathlib import Path
import re
import shutil
import signal
import socket
import statistics
import subprocess
import tempfile
import time
import venv
ROOT = Path(__file__).resolve().parents[1]
APPLICATION = '''from veela import App
import sys
app = App()
@app.route('/')
async def index(request):
return {'hello': 'world'}
if sys.argv[1] == 'embedded':
app.run(port=int(sys.argv[2]))
else:
import uvicorn
uvicorn.run(app, port=int(sys.argv[2]), host='127.0.0.1', http='h11', ws='none',
lifespan='off', loop='asyncio', interface='asgi3', workers=1, access_log=False)
'''
def parse(output):
def value(pattern):
return re.search(pattern, output, re.MULTILINE).group(1)
errors = re.search(r'Socket errors: connect (\d+), read (\d+), write (\d+), timeout (\d+)', output)
bad = re.search(r'Non-2xx or 3xx responses: (\d+)', output)
return {'requests_per_second': float(value(r'Requests/sec:\s+([\d.]+)')),
'mean_latency': value(r'^\s+Latency\s+(\S+)'),
'p50_latency': value(r'^\s+50%\s+(\S+)'),
'p99_latency': value(r'^\s+99%\s+(\S+)'),
'socket_errors': sum(map(int, errors.groups())) if errors else 0,
'non_success_responses': int(bad.group(1)) if bad else 0}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--standalone-python', required=True)
parser.add_argument('--runs', type=int, default=3)
parser.add_argument('--output', type=Path, default=ROOT / 'experiments/embedded-uvicorn/results')
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
rows = []
with tempfile.TemporaryDirectory(prefix='veela-benchmark-') as directory:
directory = Path(directory)
venv.EnvBuilder(with_pip=False).create(directory / 'venv')
embedded_python = directory / 'venv/bin/python'
shutil.copyfile(ROOT / 'veela.py', directory / 'veela.py')
(directory / 'app.py').write_text(APPLICATION)
for run in range(1, args.runs + 1):
order = ['embedded', 'standalone'] if run % 2 else ['standalone', 'embedded']
for mode in order:
with socket.socket() as sock:
sock.bind(('127.0.0.1', 0))
port = sock.getsockname()[1]
python = str(embedded_python) if mode == 'embedded' else args.standalone_python
with (args.output / f'{mode}-{run}.server.log').open('w') as log:
start = time.perf_counter()
server = subprocess.Popen([python, 'app.py', mode, str(port)], cwd=directory,
stdout=log, stderr=log)
try:
deadline = time.monotonic() + 15
while True:
if server.poll() is not None:
raise RuntimeError(f'{mode} exited during startup')
try:
conn = http.client.HTTPConnection('127.0.0.1', port, timeout=1)
conn.request('GET', '/')
response = conn.getresponse()
assert response.status == 200 and json.loads(response.read()) == {'hello': 'world'}
conn.close()
startup = (time.perf_counter() - start) * 1000
break
except OSError:
if time.monotonic() > deadline:
raise RuntimeError('Server startup timed out')
time.sleep(0.005)
url = f'http://127.0.0.1:{port}/'
print(f'{mode} {run}/{args.runs}: warmup, startup={startup:.1f}ms', flush=True)
warmup = subprocess.run(['wrk', '-t4', '-c256', '-d5s', url],
capture_output=True, text=True, check=True)
(args.output / f'{mode}-{run}.warmup.txt').write_text(warmup.stdout + warmup.stderr)
result = subprocess.run(['wrk', '-t4', '-c256', '-d20s', '--latency', url],
capture_output=True, text=True, check=True)
output = result.stdout + result.stderr
(args.output / f'{mode}-{run}.wrk.txt').write_text(output)
row = {'mode': mode, 'run': run, 'startup_ms': startup, **parse(output)}
rows.append(row)
print(json.dumps(row), flush=True)
(args.output / 'results.json').write_text(json.dumps(rows, indent=2) + '\n')
if row['socket_errors'] or row['non_success_responses']:
raise RuntimeError('Benchmark reported errors')
finally:
if server.poll() is None:
server.send_signal(signal.SIGTERM)
try:
server.wait(timeout=10)
except subprocess.TimeoutExpired:
server.kill()
server.wait()
summary = {mode: {'median_rps': statistics.median(r['requests_per_second'] for r in rows if r['mode'] == mode),
'median_startup_ms': statistics.median(r['startup_ms'] for r in rows if r['mode'] == mode)}
for mode in ('embedded', 'standalone')}
summary['embedded_to_standalone_ratio'] = summary['embedded']['median_rps'] / summary['standalone']['median_rps']
(args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
print(json.dumps(summary, indent=2), flush=True)
if __name__ == '__main__':
main()