Files
noah 5501dfd66d
main-release-gate / verify-and-release (push) Canceled after 0s
Benchmark captured BM1366 report poll path
2026-10-16 17:02:02 -06:00

148 lines
6.6 KiB
Python

#!/usr/bin/env python3
"""Generate reviewable metrics and self-contained badges from CI measurements."""
import argparse
import html
import json
import math
import pathlib
import re
import tomllib
from bench_host import identity as benchmark_host
EXPECTED_BENCHES = frozenset({
"target_matrix", "power_policy", "share_verification",
"scrypt_share_verification", "bm1366_work_encoding", "bm1362_commands",
"bm1362_job_encoding", "bm1366_report_decode", "stratum_header",
"stratum_submission", "stratum_authorize_format", "stratum_notify_parse",
"stratum_control_parse", "stratum_configure_parse", "difficulty_target",
"stratum_job_issue", "bm1366_session_cycle", "bm1362_work_encoding",
"stratum_configure_format", "stratum_subscribe_format",
"stratum_subscription_parse", "stratum_accept_parse",
"hal_socket_exchange", "stratum_socket_round_trip",
"bm13xx_crc5", "bm13xx_crc16", "bm1366_captured_report_poll",
})
BENCH_LINE = re.compile(r"\bBENCH ([A-Za-z0-9_]+) ([0-9]+(?:\.[0-9]+)?) ns/op\Z")
def coverage(path):
data = json.loads(pathlib.Path(path).read_text())
return data["data"][0]["totals"]["lines"]["percent"]
def bench_results(path):
results = {}
for line in pathlib.Path(path).read_text().splitlines():
if "BENCH " not in line:
continue
match = BENCH_LINE.search(line)
if match is None:
raise ValueError("malformed microbenchmark result")
name, value = match.groups()
measurement = float(value)
if name in results or not math.isfinite(measurement) or measurement <= 0:
raise ValueError("duplicate or invalid microbenchmark result")
results[name] = measurement
return results
def benchmark_delta(current, baseline):
if set(current) != EXPECTED_BENCHES:
raise ValueError("microbenchmark set differs from the declared suite")
if not isinstance(baseline, dict) or not set(baseline).issubset(current):
raise ValueError("baseline contains retired or invalid benchmarks")
if any(not isinstance(value, (int, float)) or isinstance(value, bool)
or not math.isfinite(value) or value <= 0 for value in baseline.values()):
raise ValueError("baseline contains invalid measurements")
if not baseline:
return None
return sum((baseline[name] - current[name]) / baseline[name]
for name in baseline) / len(baseline) * 100
def quality_grade(path):
checks = json.loads(pathlib.Path(path).read_text())
if checks != {"format": True, "clippy_warnings_denied": True}:
raise ValueError("format and strict Clippy must pass before publishing quality")
return "A"
def compact(number):
value = float(number)
units = ("", "K", "M", "G")
index = 0
while value >= 999.5 and index < len(units) - 1:
value /= 1000
index += 1
digits = 0 if value >= 100 else 1 if value >= 10 else 2
number = f"{value:.{digits}f}"
if digits:
number = number.rstrip("0").rstrip(".")
return number + units[index]
def svg(label, value, color):
label, value = html.escape(label), html.escape(value)
lw, vw = 12 + 7 * len(label), 12 + 7 * len(value)
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{lw + vw}" height="20" '
f'role="img" aria-label="{label}: {value}">'
f'<rect width="{lw}" height="20" rx="3" fill="#555"/>'
f'<rect x="{lw}" width="{vw}" height="20" rx="3" fill="{color}"/>'
f'<g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,sans-serif" font-size="11">'
f'<text x="{lw / 2}" y="14">{label}</text>'
f'<text x="{lw + vw / 2}" y="14">{value}</text></g></svg>\n')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--tests-output", required=True)
parser.add_argument("--tests-coverage", required=True)
parser.add_argument("--bench-output", required=True)
parser.add_argument("--bench-coverage", required=True)
parser.add_argument("--quality-gate", required=True)
parser.add_argument("--bench-baseline", default="badges/bench-baseline.json")
args = parser.parse_args()
test_output = pathlib.Path(args.tests_output).read_text()
tests = sum(map(int, re.findall(r"test result: ok\. (\d+) passed", test_output)))
if tests < 1:
raise SystemExit("no passing tests parsed")
current = bench_results(args.bench_output)
base_path = pathlib.Path(args.bench_baseline)
baseline = json.loads(base_path.read_text()) if base_path.exists() else {}
delta = benchmark_delta(current, baseline)
line_coverage = coverage(args.tests_coverage)
bench_coverage = coverage(args.bench_coverage)
loc = sum(len(path.read_text().splitlines()) for parent in (pathlib.Path("src"), pathlib.Path("ci"))
for path in parent.rglob("*") if path.suffix in (".rs", ".py", ".sh"))
lock = tomllib.loads(pathlib.Path("Cargo.lock").read_text())
dependencies = sum(item["name"] != "alkane-os" for item in lock["package"])
metrics = {"tests": tests, "line_coverage_pct": round(line_coverage, 1),
"bench_coverage_pct": round(bench_coverage, 1),
"bench_ns_per_op": current, "bench_improvement_pct": delta,
"bench_host": benchmark_host(),
"lines_of_code": loc, "dependencies": dependencies,
"code_quality": quality_grade(args.quality_gate),
"quality_scope": "main CI passed cargo fmt and warnings-denied Clippy on all targets",
"license": "MIT"}
pathlib.Path("badges").mkdir(exist_ok=True)
pathlib.Path("badges/metrics.json").write_text(json.dumps(metrics, indent=2) + "\n")
badges = {
"tests": ("tests", str(tests), "#4c1"),
"coverage": ("coverage", f"{line_coverage:.1f}%", "#4c1" if line_coverage >= 80 else "#dfb317"),
"bench-coverage": ("bench coverage", f"{bench_coverage:.1f}%", "#4c1" if bench_coverage >= 80 else "#dfb317"),
"bench": ("bench", "baseline set" if delta is None else f"{delta:+.1f}% vs prev", "#007ec6"),
"loc": ("lines of code", compact(loc), "#007ec6"),
"dependencies": ("dependencies", str(dependencies), "#007ec6"),
"quality": ("code quality", metrics["code_quality"],
{"A": "#4c1", "B": "#97ca00", "C": "#dfb317",
"D": "#fe7d37", "F": "#e05d44"}[metrics["code_quality"]]),
"license": ("license", "MIT", "#007ec6"),
}
for name, (label, value, color) in badges.items():
pathlib.Path(f"badges/{name}.svg").write_text(svg(label, value, color))
print(json.dumps(metrics))
if __name__ == "__main__":
main()