160 lines
7.4 KiB
Python
160 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: MIT
|
|
"""Keep measured main-branch benchmarks in the private CI artifact branch."""
|
|
|
|
import argparse
|
|
import base64
|
|
from datetime import datetime, timezone
|
|
import html
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import quote
|
|
|
|
from badge_store import BADGE_BRANCH, publish_file, request_json
|
|
from bench_host import identity as benchmark_host
|
|
|
|
INDEX = "history/index.json"
|
|
MAX_INDEX_ENTRIES = 50
|
|
|
|
|
|
def read_artifact(api: str, repo: str, token: str, filename: str):
|
|
url = f"{api}/repos/{repo}/contents/{quote(filename, safe='/')}?ref={BADGE_BRANCH}"
|
|
try:
|
|
value = request_json(url, token)
|
|
except HTTPError as error:
|
|
if error.code == 404:
|
|
return None
|
|
raise
|
|
return json.loads(base64.b64decode(value["content"]))
|
|
|
|
|
|
def valid_benches(metrics: dict) -> dict[str, float]:
|
|
values = metrics.get("bench_ns_per_op")
|
|
if not isinstance(values, dict) or not values:
|
|
raise ValueError("benchmark measurements are missing")
|
|
if any(not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_]+", key)
|
|
or not isinstance(value, (float, int)) or isinstance(value, bool)
|
|
or not math.isfinite(value) or value <= 0 for key, value in values.items()):
|
|
raise ValueError("benchmark measurements are invalid")
|
|
return values
|
|
|
|
|
|
def git_distance(ancestor: str, sha: str) -> int | None:
|
|
result = subprocess.run(("git", "rev-list", "--count", f"{ancestor}..{sha}"),
|
|
capture_output=True, text=True, check=False)
|
|
if result.returncode != 0:
|
|
return None
|
|
ancestor_check = subprocess.run(("git", "merge-base", "--is-ancestor", ancestor, sha),
|
|
capture_output=True, check=False)
|
|
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
|
|
|
|
|
|
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
|
|
distance = distance or git_distance
|
|
choices = []
|
|
for entry in index.get("entries", []):
|
|
ancestor = entry.get("sha")
|
|
if ancestor == sha or not isinstance(ancestor, str):
|
|
continue
|
|
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
|
|
continue
|
|
gap = distance(ancestor, sha)
|
|
if gap is not None and gap > 0:
|
|
choices.append((gap, entry))
|
|
if not choices:
|
|
return {}
|
|
return valid_benches(min(choices, key=lambda item: item[0])[1]["metrics"])
|
|
|
|
|
|
def updated_index(index: dict, sha: str, metrics: dict, timestamp: str) -> dict:
|
|
if not re.fullmatch(r"[0-9a-f]{40}", sha):
|
|
raise ValueError("invalid commit SHA")
|
|
valid_benches(metrics)
|
|
entries = [entry for entry in index.get("entries", []) if entry.get("sha") != sha]
|
|
entries.append({"sha": sha, "timestamp": timestamp, "metrics": metrics})
|
|
return {"schema": 1, "entries": entries[-MAX_INDEX_ENTRIES:]}
|
|
|
|
|
|
def render_html(index: dict) -> str:
|
|
entries = list(reversed(index.get("entries", [])))
|
|
names = sorted({name for entry in entries
|
|
for name in valid_benches(entry["metrics"])})
|
|
rows = []
|
|
for entry in entries:
|
|
sha = entry["sha"]
|
|
if not re.fullmatch(r"[0-9a-f]{40}", sha):
|
|
raise ValueError("invalid commit SHA in history")
|
|
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-os/commit/{sha}">{sha[:10]}</a></td>',
|
|
f'<td>{html.escape(entry["timestamp"])}</td>']
|
|
values = valid_benches(entry["metrics"])
|
|
host = entry["metrics"].get("bench_host", {})
|
|
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
|
|
cells.append(f'<td>{html.escape(str(host_label))}</td>')
|
|
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
|
|
for name in names]
|
|
rows.append("<tr>" + "".join(cells) + "</tr>")
|
|
headings = "".join(f"<th scope=\"col\">{html.escape(name)} (ns/op)</th>" for name in names)
|
|
return ("<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\">"
|
|
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"
|
|
"<title>Alkane OS benchmarks</title><style>"
|
|
"body{font:16px system-ui,sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem;color:#17212b}"
|
|
"h1{font-size:2rem}p{max-width:70ch;line-height:1.5}"
|
|
"table{border-collapse:collapse;min-width:100%}th,td{padding:.5rem;border-bottom:1px solid #ccd3da;text-align:right}"
|
|
"th:first-child,td:first-child{text-align:left}th:nth-child(2),td:nth-child(2){text-align:left}"
|
|
"th{background:#edf2f6;position:sticky;top:0}a{color:#145a9a}"
|
|
".scroll{overflow-x:auto}small{color:#55616d}</style></head><body>"
|
|
"<h1>Alkane OS microbenchmarks</h1>"
|
|
"<p>Measured on main CI. Lower ns/op is faster. Results depend on runner load and are "
|
|
"not board performance. A new benchmark has no earlier comparison.</p>"
|
|
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
|
|
"in the private ci-badges branch.</small></p>"
|
|
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
|
|
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>{headings}</tr></thead><tbody>{''.join(rows)}"
|
|
"</tbody></table></div></body></html>\n")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("action", choices=("prepare", "publish", "render"))
|
|
args = parser.parse_args()
|
|
api = os.environ["GITHUB_API_URL"].rstrip("/")
|
|
repo = os.environ["GITHUB_REPOSITORY"]
|
|
token = os.environ["GITEA_TOKEN"]
|
|
sha = os.environ["GITHUB_SHA"]
|
|
index = read_artifact(api, repo, token, INDEX) or {"schema": 1, "entries": []}
|
|
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
|
|
raise ValueError("unsupported benchmark history index")
|
|
if args.action == "prepare":
|
|
baseline = previous_measurement(index, sha, host=benchmark_host())
|
|
Path("badges").mkdir(exist_ok=True)
|
|
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
|
|
print(f"Loaded {len(baseline)} prior benchmark measurements")
|
|
return
|
|
if args.action == "render":
|
|
if not index["entries"]:
|
|
raise ValueError("no measured benchmark history to publish")
|
|
Path("bench-site").mkdir(exist_ok=True)
|
|
Path("bench-site/index.html").write_text(render_html(index))
|
|
print(f"Rendered {len(index['entries'])} measured benchmark commits")
|
|
return
|
|
metrics = json.loads(Path("badges/metrics.json").read_text())
|
|
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
index = updated_index(index, sha, metrics, timestamp)
|
|
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
|
|
publish_file(api, repo, f"history/{sha}.json", record, token,
|
|
"Record Alkane OS benchmarks", max_bytes=1_048_576)
|
|
publish_file(api, repo, INDEX, json.dumps(index, indent=2).encode() + b"\n",
|
|
token, "Update Alkane OS benchmark index", max_bytes=1_048_576)
|
|
Path("bench-site").mkdir(exist_ok=True)
|
|
Path("bench-site/index.html").write_text(render_html(index))
|
|
print(f"Recorded benchmarks for {sha[:10]}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|