Files
alkane-os/ci/bench_history.py
noah e8faf398f5
main-release-gate / verify-and-release (push) Failing after 2m30s
Publish benchmark Pages on the standing Alkane runner
2026-10-17 01:17:34 -06:00

160 lines
7.4 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Keep measured main-branch benchmarks in the private CI artifact branch."""
import argparse
import base64
from datetime import datetime, timezone
import html
import json
import math
import os
from pathlib import Path
import re
import subprocess
from urllib.error import HTTPError
from urllib.parse import quote
from badge_store import BADGE_BRANCH, publish_file, request_json
from bench_host import identity as benchmark_host
INDEX = "history/index.json"
MAX_INDEX_ENTRIES = 50
def read_artifact(api: str, repo: str, token: str, filename: str):
url = f"{api}/repos/{repo}/contents/{quote(filename, safe='/')}?ref={BADGE_BRANCH}"
try:
value = request_json(url, token)
except HTTPError as error:
if error.code == 404:
return None
raise
return json.loads(base64.b64decode(value["content"]))
def valid_benches(metrics: dict) -> dict[str, float]:
values = metrics.get("bench_ns_per_op")
if not isinstance(values, dict) or not values:
raise ValueError("benchmark measurements are missing")
if any(not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_]+", key)
or not isinstance(value, (float, int)) or isinstance(value, bool)
or not math.isfinite(value) or value <= 0 for key, value in values.items()):
raise ValueError("benchmark measurements are invalid")
return values
def git_distance(ancestor: str, sha: str) -> int | None:
result = subprocess.run(("git", "rev-list", "--count", f"{ancestor}..{sha}"),
capture_output=True, text=True, check=False)
if result.returncode != 0:
return None
ancestor_check = subprocess.run(("git", "merge-base", "--is-ancestor", ancestor, sha),
capture_output=True, check=False)
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
distance = distance or git_distance
choices = []
for entry in index.get("entries", []):
ancestor = entry.get("sha")
if ancestor == sha or not isinstance(ancestor, str):
continue
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
continue
gap = distance(ancestor, sha)
if gap is not None and gap > 0:
choices.append((gap, entry))
if not choices:
return {}
return valid_benches(min(choices, key=lambda item: item[0])[1]["metrics"])
def updated_index(index: dict, sha: str, metrics: dict, timestamp: str) -> dict:
if not re.fullmatch(r"[0-9a-f]{40}", sha):
raise ValueError("invalid commit SHA")
valid_benches(metrics)
entries = [entry for entry in index.get("entries", []) if entry.get("sha") != sha]
entries.append({"sha": sha, "timestamp": timestamp, "metrics": metrics})
return {"schema": 1, "entries": entries[-MAX_INDEX_ENTRIES:]}
def render_html(index: dict) -> str:
entries = list(reversed(index.get("entries", [])))
names = sorted({name for entry in entries
for name in valid_benches(entry["metrics"])})
rows = []
for entry in entries:
sha = entry["sha"]
if not re.fullmatch(r"[0-9a-f]{40}", sha):
raise ValueError("invalid commit SHA in history")
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-os/commit/{sha}">{sha[:10]}</a></td>',
f'<td>{html.escape(entry["timestamp"])}</td>']
values = valid_benches(entry["metrics"])
host = entry["metrics"].get("bench_host", {})
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
cells.append(f'<td>{html.escape(str(host_label))}</td>')
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
for name in names]
rows.append("<tr>" + "".join(cells) + "</tr>")
headings = "".join(f"<th scope=\"col\">{html.escape(name)} (ns/op)</th>" for name in names)
return ("<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\">"
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"
"<title>Alkane OS benchmarks</title><style>"
"body{font:16px system-ui,sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem;color:#17212b}"
"h1{font-size:2rem}p{max-width:70ch;line-height:1.5}"
"table{border-collapse:collapse;min-width:100%}th,td{padding:.5rem;border-bottom:1px solid #ccd3da;text-align:right}"
"th:first-child,td:first-child{text-align:left}th:nth-child(2),td:nth-child(2){text-align:left}"
"th{background:#edf2f6;position:sticky;top:0}a{color:#145a9a}"
".scroll{overflow-x:auto}small{color:#55616d}</style></head><body>"
"<h1>Alkane OS microbenchmarks</h1>"
"<p>Measured on main CI. Lower ns/op is faster. Results depend on runner load and are "
"not board performance. A new benchmark has no earlier comparison.</p>"
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
"in the private ci-badges branch.</small></p>"
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>{headings}</tr></thead><tbody>{''.join(rows)}"
"</tbody></table></div></body></html>\n")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=("prepare", "publish", "render"))
args = parser.parse_args()
api = os.environ["GITHUB_API_URL"].rstrip("/")
repo = os.environ["GITHUB_REPOSITORY"]
token = os.environ["GITEA_TOKEN"]
sha = os.environ["GITHUB_SHA"]
index = read_artifact(api, repo, token, INDEX) or {"schema": 1, "entries": []}
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
raise ValueError("unsupported benchmark history index")
if args.action == "prepare":
baseline = previous_measurement(index, sha, host=benchmark_host())
Path("badges").mkdir(exist_ok=True)
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
print(f"Loaded {len(baseline)} prior benchmark measurements")
return
if args.action == "render":
if not index["entries"]:
raise ValueError("no measured benchmark history to publish")
Path("bench-site").mkdir(exist_ok=True)
Path("bench-site/index.html").write_text(render_html(index))
print(f"Rendered {len(index['entries'])} measured benchmark commits")
return
metrics = json.loads(Path("badges/metrics.json").read_text())
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
index = updated_index(index, sha, metrics, timestamp)
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
publish_file(api, repo, f"history/{sha}.json", record, token,
"Record Alkane OS benchmarks", max_bytes=1_048_576)
publish_file(api, repo, INDEX, json.dumps(index, indent=2).encode() + b"\n",
token, "Update Alkane OS benchmark index", max_bytes=1_048_576)
Path("bench-site").mkdir(exist_ok=True)
Path("bench-site/index.html").write_text(render_html(index))
print(f"Recorded benchmarks for {sha[:10]}")
if __name__ == "__main__":
main()