131 lines
6.4 KiB
Python
131 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: MIT
|
|
"""Exercise benchmark history selection and safe site rendering."""
|
|
|
|
import sys
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from bench_history import (bench_badge, git_distance, previous_measurement, render_html,
|
|
same_benchmark_inputs, updated_index)
|
|
|
|
|
|
class HistoryTests(unittest.TestCase):
|
|
def test_cargo_nocapture_benchmark_lines_are_parsed(self):
|
|
output = (
|
|
"test tests::bench_verify_artifact ... bench_verify_artifact: 208.302µs\n"
|
|
"test model::tests::bench_render_cells ... bench_render_cells: "
|
|
"5000 rows x 10 cols = 8.007185ms/full-render (bytes=13553600)\n"
|
|
"bench_direct: 12ns/op\n"
|
|
)
|
|
self.assertEqual(bench_badge.parse_bench(output), {
|
|
"bench_verify_artifact": 208_302.0,
|
|
"bench_render_cells": 8_007_185.0,
|
|
"bench_direct": 12.0,
|
|
})
|
|
|
|
def test_retry_uses_previous_commit(self):
|
|
first = "a" * 40
|
|
second = "b" * 40
|
|
index = {"schema": 1, "entries": [
|
|
{"sha": first, "timestamp": "2026-01-01T00:00:00+00:00",
|
|
"metrics": {"bench_ns_per_op": {"hash": 10.0}}},
|
|
{"sha": second, "timestamp": "2026-01-02T00:00:00+00:00",
|
|
"metrics": {"bench_ns_per_op": {"hash": 8.0}}}]}
|
|
self.assertEqual(previous_measurement(index, second,
|
|
lambda ancestor, sha: 1 if ancestor == first else None),
|
|
{"hash": 10.0})
|
|
updated = updated_index(index, second, {"bench_ns_per_op": {"hash": 7.5}},
|
|
"2026-01-03T00:00:00+00:00")
|
|
self.assertEqual(len(updated["entries"]), 2)
|
|
self.assertEqual(updated["entries"][-1]["metrics"]["bench_ns_per_op"]["hash"], 7.5)
|
|
|
|
def test_invalid_measurement_and_sha_rejected(self):
|
|
self.assertIsNone(git_distance("--bad", "a" * 40))
|
|
with self.assertRaises(ValueError):
|
|
updated_index({}, "bad", {"bench_ns_per_op": {"hash": 1}}, "now")
|
|
with self.assertRaises(ValueError):
|
|
updated_index({}, "a" * 40, {"bench_ns_per_op": {"hash": -1}}, "now")
|
|
|
|
def test_baseline_must_match_cpu_class(self):
|
|
host = {"architecture": "x86_64", "cpu_model": "A", "logical_cpus": 8}
|
|
other = {"architecture": "x86_64", "cpu_model": "B", "logical_cpus": 8}
|
|
index = {"entries": [{"sha": "a" * 40, "metrics": {
|
|
"bench_ns_per_op": {"hash": 10.0}, "bench_host": host}}]}
|
|
distance = lambda _ancestor, _sha: 1
|
|
self.assertEqual(previous_measurement(index, "b" * 40, distance, host),
|
|
{"hash": 10.0})
|
|
self.assertEqual(previous_measurement(index, "b" * 40, distance, other), {})
|
|
|
|
def test_routing_change_is_marked_same_inputs_but_code_or_build_change_is_not(self):
|
|
with tempfile.TemporaryDirectory(prefix="alkane-bench-inputs-") as directory:
|
|
root = Path(directory)
|
|
workflow = root / ".gitea/workflows/release.yml"
|
|
workflow.parent.mkdir(parents=True)
|
|
code = root / "crates/alkane-gui/src/main.rs"
|
|
code.parent.mkdir(parents=True)
|
|
code.write_text("fn main() {}\n")
|
|
workflow.write_text(
|
|
"jobs:\n release:\n runs-on: alkane-verify\n"
|
|
" container:\n image: pinned-image\n"
|
|
" env:\n RUSTFLAGS: stable\n steps:\n"
|
|
" - name: Build release\n run: cargo build --release\n"
|
|
" - name: Microbenchmarks\n run: cargo test --release bench_\n")
|
|
|
|
def git(*args):
|
|
return subprocess.check_output(("git", "-C", str(root), *args),
|
|
text=True, stderr=subprocess.DEVNULL).strip()
|
|
|
|
git("init", "-q")
|
|
git("config", "user.name", "CI Test")
|
|
git("config", "user.email", "ci@example.invalid")
|
|
git("add", ".")
|
|
git("commit", "-qm", "base")
|
|
base = git("rev-parse", "HEAD")
|
|
workflow.write_text(workflow.read_text().replace("alkane-verify", "alkane-ci"))
|
|
git("commit", "-qam", "route")
|
|
route = git("rev-parse", "HEAD")
|
|
self.assertTrue(same_benchmark_inputs(base, route, root))
|
|
code.write_text("fn main() { println!(\"changed\"); }\n")
|
|
git("commit", "-qam", "code")
|
|
changed_code = git("rev-parse", "HEAD")
|
|
self.assertFalse(same_benchmark_inputs(route, changed_code, root))
|
|
workflow.write_text(workflow.read_text().replace("RUSTFLAGS: stable", "RUSTFLAGS: tuned"))
|
|
git("commit", "-qam", "flags")
|
|
flags = git("rev-parse", "HEAD")
|
|
self.assertFalse(same_benchmark_inputs(changed_code, flags, root))
|
|
|
|
def test_badge_keeps_raw_delta_and_marks_identical_inputs(self):
|
|
with tempfile.TemporaryDirectory(prefix="alkane-bench-badge-context-") as directory:
|
|
root = Path(directory)
|
|
badges = root / "badges"
|
|
badges.mkdir()
|
|
(badges / "bench-context.json").write_text(
|
|
json.dumps({"schema": 1, "same_inputs": True, "baseline_sha": "a" * 40}))
|
|
command = (sys.executable, str(Path(__file__).with_name("gen-badges.py")),
|
|
"--root", str(root), "--bench-delta=-36.4")
|
|
subprocess.run(command, check=True, capture_output=True)
|
|
self.assertIn("-36.4% same inputs", (badges / "bench.svg").read_text())
|
|
(badges / "bench-context.json").unlink()
|
|
subprocess.run(command, check=True, capture_output=True)
|
|
self.assertIn("-36.4% vs prev", (badges / "bench.svg").read_text())
|
|
|
|
def test_site_escapes_history_content(self):
|
|
entry = {"sha": "a" * 40, "timestamp": "<script>alert(1)</script>",
|
|
"metrics": {"bench_ns_per_op": {"hash": 10.0},
|
|
"bench_host": {"cpu_model": "<script>cpu</script>"},
|
|
"bench_context": {"schema": 1, "same_inputs": True}}}
|
|
page = render_html({"entries": [entry]})
|
|
self.assertIn("<script>", page)
|
|
self.assertNotIn("<script>", page)
|
|
self.assertIn("10.000", page)
|
|
self.assertIn("same inputs", page)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|