This commit is contained in:
+39
-7
@@ -4,11 +4,22 @@
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import math
|
||||
import pathlib
|
||||
import re
|
||||
import tomllib
|
||||
from bench_host import identity as benchmark_host
|
||||
|
||||
EXPECTED_BENCHES = frozenset({
|
||||
"target_matrix", "power_policy", "share_verification",
|
||||
"scrypt_share_verification", "bm1366_work_encoding", "bm1362_commands",
|
||||
"bm1362_job_encoding", "bm1366_report_decode", "stratum_header",
|
||||
"stratum_submission", "stratum_authorize_format", "stratum_notify_parse",
|
||||
"stratum_control_parse", "stratum_configure_parse", "difficulty_target",
|
||||
"stratum_job_issue", "bm1366_session_cycle",
|
||||
})
|
||||
BENCH_LINE = re.compile(r"\bBENCH ([A-Za-z0-9_]+) ([0-9]+(?:\.[0-9]+)?) ns/op\Z")
|
||||
|
||||
|
||||
def coverage(path):
|
||||
data = json.loads(pathlib.Path(path).read_text())
|
||||
@@ -16,8 +27,33 @@ def coverage(path):
|
||||
|
||||
|
||||
def bench_results(path):
|
||||
text = pathlib.Path(path).read_text()
|
||||
return {name: float(ns) for name, ns in re.findall(r"BENCH (\w+) ([0-9.]+) ns/op", text)}
|
||||
results = {}
|
||||
for line in pathlib.Path(path).read_text().splitlines():
|
||||
if "BENCH " not in line:
|
||||
continue
|
||||
match = BENCH_LINE.search(line)
|
||||
if match is None:
|
||||
raise ValueError("malformed microbenchmark result")
|
||||
name, value = match.groups()
|
||||
measurement = float(value)
|
||||
if name in results or not math.isfinite(measurement) or measurement <= 0:
|
||||
raise ValueError("duplicate or invalid microbenchmark result")
|
||||
results[name] = measurement
|
||||
return results
|
||||
|
||||
|
||||
def benchmark_delta(current, baseline):
|
||||
if set(current) != EXPECTED_BENCHES:
|
||||
raise ValueError("microbenchmark set differs from the declared suite")
|
||||
if not isinstance(baseline, dict) or not set(baseline).issubset(current):
|
||||
raise ValueError("baseline contains retired or invalid benchmarks")
|
||||
if any(not isinstance(value, (int, float)) or isinstance(value, bool)
|
||||
or not math.isfinite(value) or value <= 0 for value in baseline.values()):
|
||||
raise ValueError("baseline contains invalid measurements")
|
||||
if not baseline:
|
||||
return None
|
||||
return sum((baseline[name] - current[name]) / baseline[name]
|
||||
for name in baseline) / len(baseline) * 100
|
||||
|
||||
|
||||
def quality_grade(path):
|
||||
@@ -67,13 +103,9 @@ def main():
|
||||
if tests < 1:
|
||||
raise SystemExit("no passing tests parsed")
|
||||
current = bench_results(args.bench_output)
|
||||
if not current or any(value <= 0 for value in current.values()):
|
||||
raise SystemExit("microbenchmark result missing or zero")
|
||||
base_path = pathlib.Path(args.bench_baseline)
|
||||
baseline = json.loads(base_path.read_text()) if base_path.exists() else {}
|
||||
common = [name for name in current if name in baseline and baseline[name] > 0]
|
||||
delta = (sum((baseline[name] - current[name]) / baseline[name] for name in common)
|
||||
/ len(common) * 100) if common else None
|
||||
delta = benchmark_delta(current, baseline)
|
||||
line_coverage = coverage(args.tests_coverage)
|
||||
bench_coverage = coverage(args.bench_coverage)
|
||||
loc = sum(len(path.read_text().splitlines()) for parent in (pathlib.Path("src"), pathlib.Path("ci"))
|
||||
|
||||
+19
-2
@@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from metrics import bench_results, compact, quality_grade
|
||||
from metrics import EXPECTED_BENCHES, bench_results, benchmark_delta, compact, quality_grade
|
||||
|
||||
|
||||
class MetricTests(unittest.TestCase):
|
||||
@@ -25,10 +25,27 @@ class MetricTests(unittest.TestCase):
|
||||
def test_benchmark_parser_and_short_si_badge(self):
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "bench.txt"
|
||||
path.write_text("BENCH sha 42.125 ns/op\n")
|
||||
path.write_text("test bench_sha ... BENCH sha 42.125 ns/op\n")
|
||||
self.assertEqual(bench_results(path), {"sha": 42.125})
|
||||
path.write_text("BENCH sha 42.125 ns/op\nBENCH sha 40.000 ns/op\n")
|
||||
with self.assertRaisesRegex(ValueError, "duplicate"):
|
||||
bench_results(path)
|
||||
path.write_text("BENCH sha 42.125 ns/op\nBENCH broken nan ns/op\n")
|
||||
with self.assertRaisesRegex(ValueError, "malformed"):
|
||||
bench_results(path)
|
||||
self.assertEqual(compact(1234567), "1.23M")
|
||||
|
||||
def test_benchmark_suite_and_baseline_must_be_complete(self):
|
||||
current = {name: 10.0 for name in EXPECTED_BENCHES}
|
||||
self.assertIsNone(benchmark_delta(current, {}))
|
||||
self.assertEqual(benchmark_delta(current, {"target_matrix": 20.0}), 50.0)
|
||||
with self.assertRaisesRegex(ValueError, "declared suite"):
|
||||
benchmark_delta({"target_matrix": 10.0}, {})
|
||||
with self.assertRaisesRegex(ValueError, "retired"):
|
||||
benchmark_delta(current, {"removed_bench": 20.0})
|
||||
with self.assertRaisesRegex(ValueError, "invalid measurements"):
|
||||
benchmark_delta(current, {"target_matrix": float("inf")})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user