"""Run the instrumented FastAPI app for tail-latency diagnosis, never ranking."""
import argparse
import http.client
import json
import os
from pathlib import Path
import signal
import socket
import subprocess
import sys
import time
root = Path(__file__).resolve().parents[1]
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--server', choices=['skirr', 'peregrine'], required=True)
p.add_argument('--output', type=Path, required=True)
a = p.parse_args()
a.output.mkdir(parents=True, exist_ok=True)
with socket.socket() as s:
s.bind(('127.0.0.1', 0))
port = s.getsockname()[1]
env = dict(os.environ, PYTHONPATH=str(root/'python') + os.pathsep + str(root/'bench'),
SKIRR_TAIL_OUTPUT=str(a.output.resolve()/'timings.json'))
os.sched_setaffinity(0, {7})
command = [sys.executable, str(root/'bench/server.py'), a.server, '--app', 'tail',
'--workers', '1', '--cpus', '0', '--port', str(port)]
(a.output/'command.json').write_text(json.dumps(command, indent=2))
with (a.output/'server.log').open('w') as log:
proc = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
start_new_session=True)
try:
deadline = time.monotonic() + 10
while True:
client = http.client.HTTPConnection('127.0.0.1', port, timeout=.5)
try:
client.request('GET', '/json')
response = client.getresponse()
assert response.status == 200 and response.read() == b'{"hello":"world"}'
break
except OSError:
if proc.poll() is not None or time.monotonic() > deadline:
raise RuntimeError((a.output/'server.log').read_text())
time.sleep(.05)
finally:
client.close()
load = subprocess.run(['taskset', '-c', '4,5,6,7', 'wrk', '-t4', '-c64', '-d8s',
'--latency', f'http://127.0.0.1:{port}/json'],
capture_output=True, text=True, check=True, timeout=20)
(a.output/'diagnostic-load.txt').write_text(load.stdout)
time.sleep(1.1)
finally:
os.killpg(proc.pid, signal.SIGTERM)
proc.wait(timeout=10)
data = json.loads((a.output/'timings.json').read_text())
summary = {'counts': data['counts'], 'await_yields': data['await_yields']}
for metric, hist in data['microsecond_histograms'].items():
entries = sorted((int(us), n) for us, n in hist.items())
count = sum(n for _, n in entries)
def percentile(p):
cumulative = 0
for value, n in entries:
cumulative += n
if cumulative >= count * p:
return value
summary[metric] = dict(count=count, p50_us=percentile(.5), p99_us=percentile(.99),
max_us=entries[-1][0], total_us=sum(us*n for us, n in entries))
(a.output/'summary.json').write_text(json.dumps(summary, indent=2))
print(json.dumps(summary, indent=2))