Files
alkane-os/ci/test-metrics.py
noah 772c4aad11
main-release-gate / verify-and-release (push) Canceled after 0s
Require complete OS microbenchmark measurements
2026-10-16 05:02:18 -06:00

52 lines
2.3 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Regression checks for measured benchmark and quality badges."""
import sys
import json
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parent))
from metrics import EXPECTED_BENCHES, bench_results, benchmark_delta, compact, quality_grade
class MetricTests(unittest.TestCase):
def test_quality_requires_both_static_checks(self):
with TemporaryDirectory() as directory:
path = Path(directory) / "quality.json"
path.write_text(json.dumps({"format": True, "clippy_warnings_denied": True}))
self.assertEqual(quality_grade(path), "A")
path.write_text(json.dumps({"format": True, "clippy_warnings_denied": False}))
with self.assertRaises(ValueError):
quality_grade(path)
def test_benchmark_parser_and_short_si_badge(self):
with TemporaryDirectory() as directory:
path = Path(directory) / "bench.txt"
path.write_text("test bench_sha ... BENCH sha 42.125 ns/op\n")
self.assertEqual(bench_results(path), {"sha": 42.125})
path.write_text("BENCH sha 42.125 ns/op\nBENCH sha 40.000 ns/op\n")
with self.assertRaisesRegex(ValueError, "duplicate"):
bench_results(path)
path.write_text("BENCH sha 42.125 ns/op\nBENCH broken nan ns/op\n")
with self.assertRaisesRegex(ValueError, "malformed"):
bench_results(path)
self.assertEqual(compact(1234567), "1.23M")
def test_benchmark_suite_and_baseline_must_be_complete(self):
current = {name: 10.0 for name in EXPECTED_BENCHES}
self.assertIsNone(benchmark_delta(current, {}))
self.assertEqual(benchmark_delta(current, {"target_matrix": 20.0}), 50.0)
with self.assertRaisesRegex(ValueError, "declared suite"):
benchmark_delta({"target_matrix": 10.0}, {})
with self.assertRaisesRegex(ValueError, "retired"):
benchmark_delta(current, {"removed_bench": 20.0})
with self.assertRaisesRegex(ValueError, "invalid measurements"):
benchmark_delta(current, {"target_matrix": float("inf")})
if __name__ == "__main__":
unittest.main()