52 lines
2.3 KiB
Python
52 lines
2.3 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: MIT
|
|
"""Regression checks for measured benchmark and quality badges."""
|
|
|
|
import sys
|
|
import json
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from metrics import EXPECTED_BENCHES, bench_results, benchmark_delta, compact, quality_grade
|
|
|
|
|
|
class MetricTests(unittest.TestCase):
|
|
def test_quality_requires_both_static_checks(self):
|
|
with TemporaryDirectory() as directory:
|
|
path = Path(directory) / "quality.json"
|
|
path.write_text(json.dumps({"format": True, "clippy_warnings_denied": True}))
|
|
self.assertEqual(quality_grade(path), "A")
|
|
path.write_text(json.dumps({"format": True, "clippy_warnings_denied": False}))
|
|
with self.assertRaises(ValueError):
|
|
quality_grade(path)
|
|
|
|
def test_benchmark_parser_and_short_si_badge(self):
|
|
with TemporaryDirectory() as directory:
|
|
path = Path(directory) / "bench.txt"
|
|
path.write_text("test bench_sha ... BENCH sha 42.125 ns/op\n")
|
|
self.assertEqual(bench_results(path), {"sha": 42.125})
|
|
path.write_text("BENCH sha 42.125 ns/op\nBENCH sha 40.000 ns/op\n")
|
|
with self.assertRaisesRegex(ValueError, "duplicate"):
|
|
bench_results(path)
|
|
path.write_text("BENCH sha 42.125 ns/op\nBENCH broken nan ns/op\n")
|
|
with self.assertRaisesRegex(ValueError, "malformed"):
|
|
bench_results(path)
|
|
self.assertEqual(compact(1234567), "1.23M")
|
|
|
|
def test_benchmark_suite_and_baseline_must_be_complete(self):
|
|
current = {name: 10.0 for name in EXPECTED_BENCHES}
|
|
self.assertIsNone(benchmark_delta(current, {}))
|
|
self.assertEqual(benchmark_delta(current, {"target_matrix": 20.0}), 50.0)
|
|
with self.assertRaisesRegex(ValueError, "declared suite"):
|
|
benchmark_delta({"target_matrix": 10.0}, {})
|
|
with self.assertRaisesRegex(ValueError, "retired"):
|
|
benchmark_delta(current, {"removed_bench": 20.0})
|
|
with self.assertRaisesRegex(ValueError, "invalid measurements"):
|
|
benchmark_delta(current, {"target_matrix": float("inf")})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|