228 lines
11 KiB
Python
228 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: MIT
|
|
"""Keep measured main-branch benchmarks in the private CI artifact branch."""
|
|
|
|
import argparse
|
|
import base64
|
|
from datetime import datetime, timezone
|
|
import html
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import importlib.util
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import quote
|
|
import yaml
|
|
|
|
from badge_store import BADGE_BRANCH, publish_file, request_json
|
|
from bench_host import identity as benchmark_host
|
|
|
|
bench_spec = importlib.util.spec_from_file_location(
|
|
"bench_badge", Path(__file__).with_name("bench-badge.py"))
|
|
bench_badge = importlib.util.module_from_spec(bench_spec)
|
|
bench_spec.loader.exec_module(bench_badge)
|
|
|
|
INDEX = "history/index.json"
|
|
MAX_INDEX_ENTRIES = 50
|
|
BENCH_INPUT_PATHS = (
|
|
"Cargo.toml", "Cargo.lock", ".cargo", "crates", "thirdparty", "vendor",
|
|
"rust-toolchain.toml", "ci/bench-badge.py", "ci/bench_host.py",
|
|
"ci/ensure-build-deps.sh", "ci/ensure-wx-prefix.sh",
|
|
"ci/configure-rust-cache.sh", "ci/wx-cache-key.sh",
|
|
)
|
|
|
|
|
|
def read_artifact(api: str, repo: str, token: str, filename: str):
|
|
url = f"{api}/repos/{repo}/contents/{quote(filename, safe='/')}?ref={BADGE_BRANCH}"
|
|
try:
|
|
value = request_json(url, token)
|
|
except HTTPError as error:
|
|
if error.code == 404:
|
|
return None
|
|
raise
|
|
return json.loads(base64.b64decode(value["content"]))
|
|
|
|
|
|
def valid_benches(metrics: dict) -> dict[str, float]:
|
|
values = metrics.get("bench_ns_per_op")
|
|
if not isinstance(values, dict) or not values:
|
|
raise ValueError("benchmark measurements are missing")
|
|
if any(not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_]+", key)
|
|
or not isinstance(value, (float, int)) or isinstance(value, bool)
|
|
or not math.isfinite(value) or value <= 0 for key, value in values.items()):
|
|
raise ValueError("benchmark measurements are invalid")
|
|
return values
|
|
|
|
|
|
def git_distance(ancestor: str, sha: str) -> int | None:
|
|
if not re.fullmatch(r"[0-9a-f]{40}", ancestor) or not re.fullmatch(r"[0-9a-f]{40}", sha):
|
|
return None
|
|
result = subprocess.run(("git", "rev-list", "--count", f"{ancestor}..{sha}"),
|
|
capture_output=True, text=True, check=False)
|
|
if result.returncode != 0:
|
|
return None
|
|
ancestor_check = subprocess.run(("git", "merge-base", "--is-ancestor", ancestor, sha),
|
|
capture_output=True, check=False)
|
|
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
|
|
|
|
|
|
def previous_entry(index: dict, sha: str, distance=None, host=None) -> dict | None:
|
|
distance = distance or git_distance
|
|
choices = []
|
|
for entry in index.get("entries", []):
|
|
ancestor = entry.get("sha")
|
|
if ancestor == sha or not isinstance(ancestor, str):
|
|
continue
|
|
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
|
|
continue
|
|
gap = distance(ancestor, sha)
|
|
if gap is not None and gap > 0:
|
|
choices.append((gap, entry))
|
|
if not choices:
|
|
return None
|
|
return min(choices, key=lambda item: item[0])[1]
|
|
|
|
|
|
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
|
|
entry = previous_entry(index, sha, distance, host)
|
|
return valid_benches(entry["metrics"]) if entry else {}
|
|
|
|
|
|
def benchmark_build_settings(sha: str, root: Path | str = ".") -> dict:
|
|
source = subprocess.check_output(
|
|
("git", "show", f"{sha}:.gitea/workflows/release.yml"),
|
|
cwd=root, text=True)
|
|
job = yaml.load(source, Loader=yaml.BaseLoader)["jobs"]["release"]
|
|
commands = {step.get("name", ""): step.get("run") for step in job["steps"]}
|
|
selected = ("Install build dependencies", "Configure bounded Rust compiler cache",
|
|
"Build wxWidgets (once, then cached)", "Build release", "Microbenchmarks")
|
|
return {"image": job["container"]["image"], "env": job.get("env", {}),
|
|
"commands": {name: commands.get(name) for name in selected}}
|
|
|
|
|
|
def same_benchmark_inputs(ancestor: str, sha: str, root: Path | str = ".") -> bool:
|
|
"""Identify changes that only reroute CI without changing benchmark inputs."""
|
|
if not re.fullmatch(r"[0-9a-f]{40}", ancestor) or not re.fullmatch(r"[0-9a-f]{40}", sha):
|
|
raise ValueError("invalid benchmark comparison commit")
|
|
diff = subprocess.run(("git", "diff", "--quiet", ancestor, sha, "--",
|
|
*BENCH_INPUT_PATHS), cwd=root, check=False)
|
|
if diff.returncode not in (0, 1):
|
|
raise ValueError("cannot inspect benchmark input changes")
|
|
if diff.returncode == 1:
|
|
return False
|
|
try:
|
|
return benchmark_build_settings(ancestor, root) == benchmark_build_settings(sha, root)
|
|
except (KeyError, TypeError, subprocess.CalledProcessError, yaml.YAMLError):
|
|
return False
|
|
|
|
|
|
def updated_index(index: dict, sha: str, metrics: dict, timestamp: str) -> dict:
|
|
if not re.fullmatch(r"[0-9a-f]{40}", sha):
|
|
raise ValueError("invalid commit SHA")
|
|
valid_benches(metrics)
|
|
entries = [entry for entry in index.get("entries", []) if entry.get("sha") != sha]
|
|
entries.append({"sha": sha, "timestamp": timestamp, "metrics": metrics})
|
|
return {"schema": 1, "entries": entries[-MAX_INDEX_ENTRIES:]}
|
|
|
|
|
|
def render_html(index: dict) -> str:
|
|
entries = list(reversed(index.get("entries", [])))
|
|
names = sorted({name for entry in entries
|
|
for name in valid_benches(entry["metrics"])})
|
|
rows = []
|
|
for entry in entries:
|
|
sha = entry["sha"]
|
|
if not re.fullmatch(r"[0-9a-f]{40}", sha):
|
|
raise ValueError("invalid commit SHA in history")
|
|
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-toolkit/commit/{sha}">{sha[:10]}</a></td>',
|
|
f'<td>{html.escape(entry["timestamp"])}</td>']
|
|
values = valid_benches(entry["metrics"])
|
|
host = entry["metrics"].get("bench_host", {})
|
|
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
|
|
cells.append(f'<td>{html.escape(str(host_label))}</td>')
|
|
context = entry["metrics"].get("bench_context", {})
|
|
if isinstance(context, dict) and context.get("schema") == 1:
|
|
context_label = "same inputs" if context.get("same_inputs") else "changed inputs"
|
|
else:
|
|
context_label = "unclassified"
|
|
cells.append(f'<td>{context_label}</td>')
|
|
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
|
|
for name in names]
|
|
rows.append("<tr>" + "".join(cells) + "</tr>")
|
|
headings = "".join(f"<th scope=\"col\">{html.escape(name)} (ns/op)</th>" for name in names)
|
|
return ("<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\">"
|
|
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"
|
|
"<title>Alkane Toolkit benchmarks</title><style>"
|
|
"body{font:16px system-ui,sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem;color:#17212b}"
|
|
"h1{font-size:2rem}p{max-width:70ch;line-height:1.5}"
|
|
"table{border-collapse:collapse;min-width:100%}th,td{padding:.5rem;border-bottom:1px solid #ccd3da;text-align:right}"
|
|
"th:first-child,td:first-child{text-align:left}th:nth-child(2),td:nth-child(2){text-align:left}"
|
|
"th{background:#edf2f6;position:sticky;top:0}a{color:#145a9a}"
|
|
".scroll{overflow-x:auto}small{color:#55616d}</style></head><body>"
|
|
"<h1>Alkane Toolkit microbenchmarks</h1>"
|
|
"<p>Measured on main CI. Lower ns/op is faster. Runner load affects results. "
|
|
"Same inputs marks a repeat of the benchmark source and build settings. "
|
|
"A new benchmark has no earlier comparison.</p>"
|
|
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
|
|
"in the private ci-badges branch.</small></p>"
|
|
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
|
|
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>"
|
|
f"<th scope=\"col\">Inputs</th>{headings}</tr></thead><tbody>{''.join(rows)}"
|
|
"</tbody></table></div></body></html>\n")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("action", choices=("prepare", "publish", "render"))
|
|
args = parser.parse_args()
|
|
api = os.environ["GITHUB_API_URL"].rstrip("/")
|
|
repo = os.environ["GITHUB_REPOSITORY"]
|
|
token = os.environ["GITEA_TOKEN"]
|
|
sha = os.environ["GITHUB_SHA"]
|
|
index = read_artifact(api, repo, token, INDEX) or {"schema": 1, "entries": []}
|
|
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
|
|
raise ValueError("unsupported benchmark history index")
|
|
if args.action == "prepare":
|
|
entry = previous_entry(index, sha, host=benchmark_host())
|
|
baseline = valid_benches(entry["metrics"]) if entry else {}
|
|
same_inputs = same_benchmark_inputs(entry["sha"], sha) if entry else False
|
|
Path("badges").mkdir(exist_ok=True)
|
|
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
|
|
Path("badges/bench-context.json").write_text(
|
|
json.dumps({"schema": 1, "same_inputs": same_inputs,
|
|
"baseline_sha": entry["sha"] if entry else None}) + "\n")
|
|
print(f"Loaded {len(baseline)} prior benchmark measurements; "
|
|
f"same inputs: {str(same_inputs).lower()}")
|
|
return
|
|
if args.action == "render":
|
|
if not index["entries"]:
|
|
raise ValueError("no measured benchmark history to publish")
|
|
Path("bench-site").mkdir(exist_ok=True)
|
|
Path("bench-site/index.html").write_text(render_html(index))
|
|
print(f"Rendered {len(index['entries'])} measured benchmark commits")
|
|
return
|
|
context = json.loads(Path("badges/bench-context.json").read_text())
|
|
if (not isinstance(context, dict) or context.get("schema") != 1 or
|
|
not isinstance(context.get("same_inputs"), bool)):
|
|
raise ValueError("invalid benchmark comparison context")
|
|
metrics = {"bench_ns_per_op": bench_badge.require_suite(bench_badge.parse_bench(
|
|
Path("bench-current.txt").read_text())), "bench_host": benchmark_host(),
|
|
"bench_context": context}
|
|
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
index = updated_index(index, sha, metrics, timestamp)
|
|
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
|
|
publish_file(api, repo, f"history/{sha}.json", record, token,
|
|
"Record Alkane Toolkit benchmarks", max_bytes=1_048_576)
|
|
publish_file(api, repo, INDEX, json.dumps(index, indent=2).encode() + b"\n",
|
|
token, "Update Alkane Toolkit benchmark index", max_bytes=1_048_576)
|
|
Path("bench-site").mkdir(exist_ok=True)
|
|
Path("bench-site/index.html").write_text(render_html(index))
|
|
print(f"Recorded benchmarks for {sha[:10]}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|