Require complete OS microbenchmark measurements
main-release-gate / verify-and-release (push) Canceled after 0s

This commit is contained in:
2026-10-16 05:02:18 -06:00
parent 79c6c72fd6
commit 772c4aad11
2 changed files with 58 additions and 9 deletions
+39 -7
View File
@@ -4,11 +4,22 @@
import argparse
import html
import json
import math
import pathlib
import re
import tomllib
from bench_host import identity as benchmark_host
EXPECTED_BENCHES = frozenset({
"target_matrix", "power_policy", "share_verification",
"scrypt_share_verification", "bm1366_work_encoding", "bm1362_commands",
"bm1362_job_encoding", "bm1366_report_decode", "stratum_header",
"stratum_submission", "stratum_authorize_format", "stratum_notify_parse",
"stratum_control_parse", "stratum_configure_parse", "difficulty_target",
"stratum_job_issue", "bm1366_session_cycle",
})
BENCH_LINE = re.compile(r"\bBENCH ([A-Za-z0-9_]+) ([0-9]+(?:\.[0-9]+)?) ns/op\Z")
def coverage(path):
data = json.loads(pathlib.Path(path).read_text())
@@ -16,8 +27,33 @@ def coverage(path):
def bench_results(path):
text = pathlib.Path(path).read_text()
return {name: float(ns) for name, ns in re.findall(r"BENCH (\w+) ([0-9.]+) ns/op", text)}
results = {}
for line in pathlib.Path(path).read_text().splitlines():
if "BENCH " not in line:
continue
match = BENCH_LINE.search(line)
if match is None:
raise ValueError("malformed microbenchmark result")
name, value = match.groups()
measurement = float(value)
if name in results or not math.isfinite(measurement) or measurement <= 0:
raise ValueError("duplicate or invalid microbenchmark result")
results[name] = measurement
return results
def benchmark_delta(current, baseline):
if set(current) != EXPECTED_BENCHES:
raise ValueError("microbenchmark set differs from the declared suite")
if not isinstance(baseline, dict) or not set(baseline).issubset(current):
raise ValueError("baseline contains retired or invalid benchmarks")
if any(not isinstance(value, (int, float)) or isinstance(value, bool)
or not math.isfinite(value) or value <= 0 for value in baseline.values()):
raise ValueError("baseline contains invalid measurements")
if not baseline:
return None
return sum((baseline[name] - current[name]) / baseline[name]
for name in baseline) / len(baseline) * 100
def quality_grade(path):
@@ -67,13 +103,9 @@ def main():
if tests < 1:
raise SystemExit("no passing tests parsed")
current = bench_results(args.bench_output)
if not current or any(value <= 0 for value in current.values()):
raise SystemExit("microbenchmark result missing or zero")
base_path = pathlib.Path(args.bench_baseline)
baseline = json.loads(base_path.read_text()) if base_path.exists() else {}
common = [name for name in current if name in baseline and baseline[name] > 0]
delta = (sum((baseline[name] - current[name]) / baseline[name] for name in common)
/ len(common) * 100) if common else None
delta = benchmark_delta(current, baseline)
line_coverage = coverage(args.tests_coverage)
bench_coverage = coverage(args.bench_coverage)
loc = sum(len(path.read_text().splitlines()) for parent in (pathlib.Path("src"), pathlib.Path("ci"))
+19 -2
View File
@@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parent))
from metrics import bench_results, compact, quality_grade
from metrics import EXPECTED_BENCHES, bench_results, benchmark_delta, compact, quality_grade
class MetricTests(unittest.TestCase):
@@ -25,10 +25,27 @@ class MetricTests(unittest.TestCase):
def test_benchmark_parser_and_short_si_badge(self):
with TemporaryDirectory() as directory:
path = Path(directory) / "bench.txt"
path.write_text("BENCH sha 42.125 ns/op\n")
path.write_text("test bench_sha ... BENCH sha 42.125 ns/op\n")
self.assertEqual(bench_results(path), {"sha": 42.125})
path.write_text("BENCH sha 42.125 ns/op\nBENCH sha 40.000 ns/op\n")
with self.assertRaisesRegex(ValueError, "duplicate"):
bench_results(path)
path.write_text("BENCH sha 42.125 ns/op\nBENCH broken nan ns/op\n")
with self.assertRaisesRegex(ValueError, "malformed"):
bench_results(path)
self.assertEqual(compact(1234567), "1.23M")
def test_benchmark_suite_and_baseline_must_be_complete(self):
current = {name: 10.0 for name in EXPECTED_BENCHES}
self.assertIsNone(benchmark_delta(current, {}))
self.assertEqual(benchmark_delta(current, {"target_matrix": 20.0}), 50.0)
with self.assertRaisesRegex(ValueError, "declared suite"):
benchmark_delta({"target_matrix": 10.0}, {})
with self.assertRaisesRegex(ValueError, "retired"):
benchmark_delta(current, {"removed_bench": 20.0})
with self.assertRaisesRegex(ValueError, "invalid measurements"):
benchmark_delta(current, {"target_matrix": float("inf")})
if __name__ == "__main__":
unittest.main()