Files

228 lines
11 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Keep measured main-branch benchmarks in the private CI artifact branch."""
import argparse
import base64
from datetime import datetime, timezone
import html
import json
import math
import os
from pathlib import Path
import re
import subprocess
import importlib.util
from urllib.error import HTTPError
from urllib.parse import quote
import yaml
from badge_store import BADGE_BRANCH, publish_file, request_json
from bench_host import identity as benchmark_host
bench_spec = importlib.util.spec_from_file_location(
"bench_badge", Path(__file__).with_name("bench-badge.py"))
bench_badge = importlib.util.module_from_spec(bench_spec)
bench_spec.loader.exec_module(bench_badge)
INDEX = "history/index.json"
MAX_INDEX_ENTRIES = 50
BENCH_INPUT_PATHS = (
"Cargo.toml", "Cargo.lock", ".cargo", "crates", "thirdparty", "vendor",
"rust-toolchain.toml", "ci/bench-badge.py", "ci/bench_host.py",
"ci/ensure-build-deps.sh", "ci/ensure-wx-prefix.sh",
"ci/configure-rust-cache.sh", "ci/wx-cache-key.sh",
)
def read_artifact(api: str, repo: str, token: str, filename: str):
url = f"{api}/repos/{repo}/contents/{quote(filename, safe='/')}?ref={BADGE_BRANCH}"
try:
value = request_json(url, token)
except HTTPError as error:
if error.code == 404:
return None
raise
return json.loads(base64.b64decode(value["content"]))
def valid_benches(metrics: dict) -> dict[str, float]:
values = metrics.get("bench_ns_per_op")
if not isinstance(values, dict) or not values:
raise ValueError("benchmark measurements are missing")
if any(not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_]+", key)
or not isinstance(value, (float, int)) or isinstance(value, bool)
or not math.isfinite(value) or value <= 0 for key, value in values.items()):
raise ValueError("benchmark measurements are invalid")
return values
def git_distance(ancestor: str, sha: str) -> int | None:
if not re.fullmatch(r"[0-9a-f]{40}", ancestor) or not re.fullmatch(r"[0-9a-f]{40}", sha):
return None
result = subprocess.run(("git", "rev-list", "--count", f"{ancestor}..{sha}"),
capture_output=True, text=True, check=False)
if result.returncode != 0:
return None
ancestor_check = subprocess.run(("git", "merge-base", "--is-ancestor", ancestor, sha),
capture_output=True, check=False)
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
def previous_entry(index: dict, sha: str, distance=None, host=None) -> dict | None:
distance = distance or git_distance
choices = []
for entry in index.get("entries", []):
ancestor = entry.get("sha")
if ancestor == sha or not isinstance(ancestor, str):
continue
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
continue
gap = distance(ancestor, sha)
if gap is not None and gap > 0:
choices.append((gap, entry))
if not choices:
return None
return min(choices, key=lambda item: item[0])[1]
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
entry = previous_entry(index, sha, distance, host)
return valid_benches(entry["metrics"]) if entry else {}
def benchmark_build_settings(sha: str, root: Path | str = ".") -> dict:
source = subprocess.check_output(
("git", "show", f"{sha}:.gitea/workflows/release.yml"),
cwd=root, text=True)
job = yaml.load(source, Loader=yaml.BaseLoader)["jobs"]["release"]
commands = {step.get("name", ""): step.get("run") for step in job["steps"]}
selected = ("Install build dependencies", "Configure bounded Rust compiler cache",
"Build wxWidgets (once, then cached)", "Build release", "Microbenchmarks")
return {"image": job["container"]["image"], "env": job.get("env", {}),
"commands": {name: commands.get(name) for name in selected}}
def same_benchmark_inputs(ancestor: str, sha: str, root: Path | str = ".") -> bool:
"""Identify changes that only reroute CI without changing benchmark inputs."""
if not re.fullmatch(r"[0-9a-f]{40}", ancestor) or not re.fullmatch(r"[0-9a-f]{40}", sha):
raise ValueError("invalid benchmark comparison commit")
diff = subprocess.run(("git", "diff", "--quiet", ancestor, sha, "--",
*BENCH_INPUT_PATHS), cwd=root, check=False)
if diff.returncode not in (0, 1):
raise ValueError("cannot inspect benchmark input changes")
if diff.returncode == 1:
return False
try:
return benchmark_build_settings(ancestor, root) == benchmark_build_settings(sha, root)
except (KeyError, TypeError, subprocess.CalledProcessError, yaml.YAMLError):
return False
def updated_index(index: dict, sha: str, metrics: dict, timestamp: str) -> dict:
if not re.fullmatch(r"[0-9a-f]{40}", sha):
raise ValueError("invalid commit SHA")
valid_benches(metrics)
entries = [entry for entry in index.get("entries", []) if entry.get("sha") != sha]
entries.append({"sha": sha, "timestamp": timestamp, "metrics": metrics})
return {"schema": 1, "entries": entries[-MAX_INDEX_ENTRIES:]}
def render_html(index: dict) -> str:
entries = list(reversed(index.get("entries", [])))
names = sorted({name for entry in entries
for name in valid_benches(entry["metrics"])})
rows = []
for entry in entries:
sha = entry["sha"]
if not re.fullmatch(r"[0-9a-f]{40}", sha):
raise ValueError("invalid commit SHA in history")
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-toolkit/commit/{sha}">{sha[:10]}</a></td>',
f'<td>{html.escape(entry["timestamp"])}</td>']
values = valid_benches(entry["metrics"])
host = entry["metrics"].get("bench_host", {})
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
cells.append(f'<td>{html.escape(str(host_label))}</td>')
context = entry["metrics"].get("bench_context", {})
if isinstance(context, dict) and context.get("schema") == 1:
context_label = "same inputs" if context.get("same_inputs") else "changed inputs"
else:
context_label = "unclassified"
cells.append(f'<td>{context_label}</td>')
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
for name in names]
rows.append("<tr>" + "".join(cells) + "</tr>")
headings = "".join(f"<th scope=\"col\">{html.escape(name)} (ns/op)</th>" for name in names)
return ("<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\">"
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"
"<title>Alkane Toolkit benchmarks</title><style>"
"body{font:16px system-ui,sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem;color:#17212b}"
"h1{font-size:2rem}p{max-width:70ch;line-height:1.5}"
"table{border-collapse:collapse;min-width:100%}th,td{padding:.5rem;border-bottom:1px solid #ccd3da;text-align:right}"
"th:first-child,td:first-child{text-align:left}th:nth-child(2),td:nth-child(2){text-align:left}"
"th{background:#edf2f6;position:sticky;top:0}a{color:#145a9a}"
".scroll{overflow-x:auto}small{color:#55616d}</style></head><body>"
"<h1>Alkane Toolkit microbenchmarks</h1>"
"<p>Measured on main CI. Lower ns/op is faster. Runner load affects results. "
"Same inputs marks a repeat of the benchmark source and build settings. "
"A new benchmark has no earlier comparison.</p>"
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
"in the private ci-badges branch.</small></p>"
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>"
f"<th scope=\"col\">Inputs</th>{headings}</tr></thead><tbody>{''.join(rows)}"
"</tbody></table></div></body></html>\n")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=("prepare", "publish", "render"))
args = parser.parse_args()
api = os.environ["GITHUB_API_URL"].rstrip("/")
repo = os.environ["GITHUB_REPOSITORY"]
token = os.environ["GITEA_TOKEN"]
sha = os.environ["GITHUB_SHA"]
index = read_artifact(api, repo, token, INDEX) or {"schema": 1, "entries": []}
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
raise ValueError("unsupported benchmark history index")
if args.action == "prepare":
entry = previous_entry(index, sha, host=benchmark_host())
baseline = valid_benches(entry["metrics"]) if entry else {}
same_inputs = same_benchmark_inputs(entry["sha"], sha) if entry else False
Path("badges").mkdir(exist_ok=True)
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
Path("badges/bench-context.json").write_text(
json.dumps({"schema": 1, "same_inputs": same_inputs,
"baseline_sha": entry["sha"] if entry else None}) + "\n")
print(f"Loaded {len(baseline)} prior benchmark measurements; "
f"same inputs: {str(same_inputs).lower()}")
return
if args.action == "render":
if not index["entries"]:
raise ValueError("no measured benchmark history to publish")
Path("bench-site").mkdir(exist_ok=True)
Path("bench-site/index.html").write_text(render_html(index))
print(f"Rendered {len(index['entries'])} measured benchmark commits")
return
context = json.loads(Path("badges/bench-context.json").read_text())
if (not isinstance(context, dict) or context.get("schema") != 1 or
not isinstance(context.get("same_inputs"), bool)):
raise ValueError("invalid benchmark comparison context")
metrics = {"bench_ns_per_op": bench_badge.require_suite(bench_badge.parse_bench(
Path("bench-current.txt").read_text())), "bench_host": benchmark_host(),
"bench_context": context}
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
index = updated_index(index, sha, metrics, timestamp)
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
publish_file(api, repo, f"history/{sha}.json", record, token,
"Record Alkane Toolkit benchmarks", max_bytes=1_048_576)
publish_file(api, repo, INDEX, json.dumps(index, indent=2).encode() + b"\n",
token, "Update Alkane Toolkit benchmark index", max_bytes=1_048_576)
Path("bench-site").mkdir(exist_ok=True)
Path("bench-site/index.html").write_text(render_html(index))
print(f"Recorded benchmarks for {sha[:10]}")
if __name__ == "__main__":
main()