148 lines
6.6 KiB
Python
148 lines
6.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate reviewable metrics and self-contained badges from CI measurements."""
|
|
|
|
import argparse
|
|
import html
|
|
import json
|
|
import math
|
|
import pathlib
|
|
import re
|
|
import tomllib
|
|
from bench_host import identity as benchmark_host
|
|
|
|
EXPECTED_BENCHES = frozenset({
|
|
"target_matrix", "power_policy", "share_verification",
|
|
"scrypt_share_verification", "bm1366_work_encoding", "bm1362_commands",
|
|
"bm1362_job_encoding", "bm1366_report_decode", "stratum_header",
|
|
"stratum_submission", "stratum_authorize_format", "stratum_notify_parse",
|
|
"stratum_control_parse", "stratum_configure_parse", "difficulty_target",
|
|
"stratum_job_issue", "bm1366_session_cycle", "bm1362_work_encoding",
|
|
"stratum_configure_format", "stratum_subscribe_format",
|
|
"stratum_subscription_parse", "stratum_accept_parse",
|
|
"hal_socket_exchange", "stratum_socket_round_trip",
|
|
"bm13xx_crc5", "bm13xx_crc16", "bm1366_captured_report_poll",
|
|
})
|
|
BENCH_LINE = re.compile(r"\bBENCH ([A-Za-z0-9_]+) ([0-9]+(?:\.[0-9]+)?) ns/op\Z")
|
|
|
|
|
|
def coverage(path):
|
|
data = json.loads(pathlib.Path(path).read_text())
|
|
return data["data"][0]["totals"]["lines"]["percent"]
|
|
|
|
|
|
def bench_results(path):
|
|
results = {}
|
|
for line in pathlib.Path(path).read_text().splitlines():
|
|
if "BENCH " not in line:
|
|
continue
|
|
match = BENCH_LINE.search(line)
|
|
if match is None:
|
|
raise ValueError("malformed microbenchmark result")
|
|
name, value = match.groups()
|
|
measurement = float(value)
|
|
if name in results or not math.isfinite(measurement) or measurement <= 0:
|
|
raise ValueError("duplicate or invalid microbenchmark result")
|
|
results[name] = measurement
|
|
return results
|
|
|
|
|
|
def benchmark_delta(current, baseline):
|
|
if set(current) != EXPECTED_BENCHES:
|
|
raise ValueError("microbenchmark set differs from the declared suite")
|
|
if not isinstance(baseline, dict) or not set(baseline).issubset(current):
|
|
raise ValueError("baseline contains retired or invalid benchmarks")
|
|
if any(not isinstance(value, (int, float)) or isinstance(value, bool)
|
|
or not math.isfinite(value) or value <= 0 for value in baseline.values()):
|
|
raise ValueError("baseline contains invalid measurements")
|
|
if not baseline:
|
|
return None
|
|
return sum((baseline[name] - current[name]) / baseline[name]
|
|
for name in baseline) / len(baseline) * 100
|
|
|
|
|
|
def quality_grade(path):
|
|
checks = json.loads(pathlib.Path(path).read_text())
|
|
if checks != {"format": True, "clippy_warnings_denied": True}:
|
|
raise ValueError("format and strict Clippy must pass before publishing quality")
|
|
return "A"
|
|
|
|
|
|
def compact(number):
|
|
value = float(number)
|
|
units = ("", "K", "M", "G")
|
|
index = 0
|
|
while value >= 999.5 and index < len(units) - 1:
|
|
value /= 1000
|
|
index += 1
|
|
digits = 0 if value >= 100 else 1 if value >= 10 else 2
|
|
number = f"{value:.{digits}f}"
|
|
if digits:
|
|
number = number.rstrip("0").rstrip(".")
|
|
return number + units[index]
|
|
|
|
|
|
def svg(label, value, color):
|
|
label, value = html.escape(label), html.escape(value)
|
|
lw, vw = 12 + 7 * len(label), 12 + 7 * len(value)
|
|
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{lw + vw}" height="20" '
|
|
f'role="img" aria-label="{label}: {value}">'
|
|
f'<rect width="{lw}" height="20" rx="3" fill="#555"/>'
|
|
f'<rect x="{lw}" width="{vw}" height="20" rx="3" fill="{color}"/>'
|
|
f'<g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,sans-serif" font-size="11">'
|
|
f'<text x="{lw / 2}" y="14">{label}</text>'
|
|
f'<text x="{lw + vw / 2}" y="14">{value}</text></g></svg>\n')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--tests-output", required=True)
|
|
parser.add_argument("--tests-coverage", required=True)
|
|
parser.add_argument("--bench-output", required=True)
|
|
parser.add_argument("--bench-coverage", required=True)
|
|
parser.add_argument("--quality-gate", required=True)
|
|
parser.add_argument("--bench-baseline", default="badges/bench-baseline.json")
|
|
args = parser.parse_args()
|
|
test_output = pathlib.Path(args.tests_output).read_text()
|
|
tests = sum(map(int, re.findall(r"test result: ok\. (\d+) passed", test_output)))
|
|
if tests < 1:
|
|
raise SystemExit("no passing tests parsed")
|
|
current = bench_results(args.bench_output)
|
|
base_path = pathlib.Path(args.bench_baseline)
|
|
baseline = json.loads(base_path.read_text()) if base_path.exists() else {}
|
|
delta = benchmark_delta(current, baseline)
|
|
line_coverage = coverage(args.tests_coverage)
|
|
bench_coverage = coverage(args.bench_coverage)
|
|
loc = sum(len(path.read_text().splitlines()) for parent in (pathlib.Path("src"), pathlib.Path("ci"))
|
|
for path in parent.rglob("*") if path.suffix in (".rs", ".py", ".sh"))
|
|
lock = tomllib.loads(pathlib.Path("Cargo.lock").read_text())
|
|
dependencies = sum(item["name"] != "alkane-os" for item in lock["package"])
|
|
metrics = {"tests": tests, "line_coverage_pct": round(line_coverage, 1),
|
|
"bench_coverage_pct": round(bench_coverage, 1),
|
|
"bench_ns_per_op": current, "bench_improvement_pct": delta,
|
|
"bench_host": benchmark_host(),
|
|
"lines_of_code": loc, "dependencies": dependencies,
|
|
"code_quality": quality_grade(args.quality_gate),
|
|
"quality_scope": "main CI passed cargo fmt and warnings-denied Clippy on all targets",
|
|
"license": "MIT"}
|
|
pathlib.Path("badges").mkdir(exist_ok=True)
|
|
pathlib.Path("badges/metrics.json").write_text(json.dumps(metrics, indent=2) + "\n")
|
|
badges = {
|
|
"tests": ("tests", str(tests), "#4c1"),
|
|
"coverage": ("coverage", f"{line_coverage:.1f}%", "#4c1" if line_coverage >= 80 else "#dfb317"),
|
|
"bench-coverage": ("bench coverage", f"{bench_coverage:.1f}%", "#4c1" if bench_coverage >= 80 else "#dfb317"),
|
|
"bench": ("bench", "baseline set" if delta is None else f"{delta:+.1f}% vs prev", "#007ec6"),
|
|
"loc": ("lines of code", compact(loc), "#007ec6"),
|
|
"dependencies": ("dependencies", str(dependencies), "#007ec6"),
|
|
"quality": ("code quality", metrics["code_quality"],
|
|
{"A": "#4c1", "B": "#97ca00", "C": "#dfb317",
|
|
"D": "#fe7d37", "F": "#e05d44"}[metrics["code_quality"]]),
|
|
"license": ("license", "MIT", "#007ec6"),
|
|
}
|
|
for name, (label, value, color) in badges.items():
|
|
pathlib.Path(f"badges/{name}.svg").write_text(svg(label, value, color))
|
|
print(json.dumps(metrics))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|