Files
alkane-toolkit/ci/test-bench-history.py

131 lines
6.4 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Exercise benchmark history selection and safe site rendering."""
import sys
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parent))
from bench_history import (bench_badge, git_distance, previous_measurement, render_html,
same_benchmark_inputs, updated_index)
class HistoryTests(unittest.TestCase):
def test_cargo_nocapture_benchmark_lines_are_parsed(self):
output = (
"test tests::bench_verify_artifact ... bench_verify_artifact: 208.302µs\n"
"test model::tests::bench_render_cells ... bench_render_cells: "
"5000 rows x 10 cols = 8.007185ms/full-render (bytes=13553600)\n"
"bench_direct: 12ns/op\n"
)
self.assertEqual(bench_badge.parse_bench(output), {
"bench_verify_artifact": 208_302.0,
"bench_render_cells": 8_007_185.0,
"bench_direct": 12.0,
})
def test_retry_uses_previous_commit(self):
first = "a" * 40
second = "b" * 40
index = {"schema": 1, "entries": [
{"sha": first, "timestamp": "2026-01-01T00:00:00+00:00",
"metrics": {"bench_ns_per_op": {"hash": 10.0}}},
{"sha": second, "timestamp": "2026-01-02T00:00:00+00:00",
"metrics": {"bench_ns_per_op": {"hash": 8.0}}}]}
self.assertEqual(previous_measurement(index, second,
lambda ancestor, sha: 1 if ancestor == first else None),
{"hash": 10.0})
updated = updated_index(index, second, {"bench_ns_per_op": {"hash": 7.5}},
"2026-01-03T00:00:00+00:00")
self.assertEqual(len(updated["entries"]), 2)
self.assertEqual(updated["entries"][-1]["metrics"]["bench_ns_per_op"]["hash"], 7.5)
def test_invalid_measurement_and_sha_rejected(self):
self.assertIsNone(git_distance("--bad", "a" * 40))
with self.assertRaises(ValueError):
updated_index({}, "bad", {"bench_ns_per_op": {"hash": 1}}, "now")
with self.assertRaises(ValueError):
updated_index({}, "a" * 40, {"bench_ns_per_op": {"hash": -1}}, "now")
def test_baseline_must_match_cpu_class(self):
host = {"architecture": "x86_64", "cpu_model": "A", "logical_cpus": 8}
other = {"architecture": "x86_64", "cpu_model": "B", "logical_cpus": 8}
index = {"entries": [{"sha": "a" * 40, "metrics": {
"bench_ns_per_op": {"hash": 10.0}, "bench_host": host}}]}
distance = lambda _ancestor, _sha: 1
self.assertEqual(previous_measurement(index, "b" * 40, distance, host),
{"hash": 10.0})
self.assertEqual(previous_measurement(index, "b" * 40, distance, other), {})
def test_routing_change_is_marked_same_inputs_but_code_or_build_change_is_not(self):
with tempfile.TemporaryDirectory(prefix="alkane-bench-inputs-") as directory:
root = Path(directory)
workflow = root / ".gitea/workflows/release.yml"
workflow.parent.mkdir(parents=True)
code = root / "crates/alkane-gui/src/main.rs"
code.parent.mkdir(parents=True)
code.write_text("fn main() {}\n")
workflow.write_text(
"jobs:\n release:\n runs-on: alkane-verify\n"
" container:\n image: pinned-image\n"
" env:\n RUSTFLAGS: stable\n steps:\n"
" - name: Build release\n run: cargo build --release\n"
" - name: Microbenchmarks\n run: cargo test --release bench_\n")
def git(*args):
return subprocess.check_output(("git", "-C", str(root), *args),
text=True, stderr=subprocess.DEVNULL).strip()
git("init", "-q")
git("config", "user.name", "CI Test")
git("config", "user.email", "ci@example.invalid")
git("add", ".")
git("commit", "-qm", "base")
base = git("rev-parse", "HEAD")
workflow.write_text(workflow.read_text().replace("alkane-verify", "alkane-ci"))
git("commit", "-qam", "route")
route = git("rev-parse", "HEAD")
self.assertTrue(same_benchmark_inputs(base, route, root))
code.write_text("fn main() { println!(\"changed\"); }\n")
git("commit", "-qam", "code")
changed_code = git("rev-parse", "HEAD")
self.assertFalse(same_benchmark_inputs(route, changed_code, root))
workflow.write_text(workflow.read_text().replace("RUSTFLAGS: stable", "RUSTFLAGS: tuned"))
git("commit", "-qam", "flags")
flags = git("rev-parse", "HEAD")
self.assertFalse(same_benchmark_inputs(changed_code, flags, root))
def test_badge_keeps_raw_delta_and_marks_identical_inputs(self):
with tempfile.TemporaryDirectory(prefix="alkane-bench-badge-context-") as directory:
root = Path(directory)
badges = root / "badges"
badges.mkdir()
(badges / "bench-context.json").write_text(
json.dumps({"schema": 1, "same_inputs": True, "baseline_sha": "a" * 40}))
command = (sys.executable, str(Path(__file__).with_name("gen-badges.py")),
"--root", str(root), "--bench-delta=-36.4")
subprocess.run(command, check=True, capture_output=True)
self.assertIn("-36.4% same inputs", (badges / "bench.svg").read_text())
(badges / "bench-context.json").unlink()
subprocess.run(command, check=True, capture_output=True)
self.assertIn("-36.4% vs prev", (badges / "bench.svg").read_text())
def test_site_escapes_history_content(self):
entry = {"sha": "a" * 40, "timestamp": "<script>alert(1)</script>",
"metrics": {"bench_ns_per_op": {"hash": 10.0},
"bench_host": {"cpu_model": "<script>cpu</script>"},
"bench_context": {"schema": 1, "same_inputs": True}}}
page = render_html({"entries": [entry]})
self.assertIn("&lt;script&gt;", page)
self.assertNotIn("<script>", page)
self.assertIn("10.000", page)
self.assertIn("same inputs", page)
if __name__ == "__main__":
unittest.main()