Require complete Toolkit benchmarks before release
build-test-release / release (push) Successful in 16m41s

This commit is contained in:
2026-10-16 05:11:44 -06:00
parent ba782e7895
commit a1f610a393
4 changed files with 90 additions and 43 deletions
+6 -3
View File
@@ -38,6 +38,9 @@ jobs:
- name: Benchmark history checks
run: python3 ci/test-bench-history.py
- name: Benchmark suite checks
run: python3 ci/test-bench-badge.py
- name: Install build dependencies
run: |
apt-get update -qq
@@ -89,12 +92,12 @@ jobs:
- name: Microbenchmarks
run: |
# Capture the bench stdout for the weighted-average delta badge (round 19
# #1) while keeping the step a hard gate: pipefail preserves cargo's exit
# code (each bench asserts its own absolute ceiling) even through tee.
# Require every declared benchmark before publishing history or a release.
set -o pipefail
cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ --test-threads=1 \
| tee bench-current.txt
python3 ci/bench-badge.py --current bench-current.txt \
--baseline badges/bench-baseline.json --no-write
- name: Record private benchmark history
env:
+40 -38
View File
@@ -1,27 +1,8 @@
#!/usr/bin/env python3
"""Compute the weighted-average microbenchmark delta versus the previous release.
Reads the captured stdout of `cargo test ... --ignored --nocapture bench_` (whose
timings are Rust `Duration` Debug strings: "12.3ms", "512us"/"512µs", "840ns",
"1.2s"), compares each bench to a committed baseline of the previous release's
numbers, and prints the time-weighted average change as a signed percentage:
(sum(current) / sum(baseline) - 1) * 100 over benches present in both
Weighting by duration means the slow benches dominate, so the number tracks
overall wall-time change rather than giving a tiny bench equal say. Then the
baseline is rewritten with the current numbers, so the next release compares
against this one ("the previous commit").
Usage:
bench-badge.py --current bench-current.txt --baseline badges/bench-baseline.json
Prints the delta as a plain number (e.g. "-3.2", or "NEW" when there is no
usable baseline) to stdout. The baseline JSON lives under badges/ so committing
it back does not trip compute-version.sh's path-based release bump.
"""
"""Check the Toolkit benchmark suite and compare it with the prior release."""
import argparse
import json
import math
import os
import re
import sys
@@ -29,9 +10,17 @@ import sys
# A Duration Debug token: a number immediately followed by a time unit, with the
# multi-char units listed before the bare "s" so alternation prefers them. The
# trailing (?![a-zA-Z]) stops "s" in words like "rows"/"cidrs" from matching.
_DUR = re.compile(r"([0-9]+(?:\.[0-9]+)?)\s*(ns|µs|us|ms|s)(?![a-zA-Z])")
_UNIT_NS = {"ns": 1.0, "us": 1e3, "µs": 1e3, "ms": 1e6, "s": 1e9}
_DUR = re.compile(r"([0-9]+(?:\.[0-9]+)?)\s*(ns|\u00b5s|us|ms|s)(?![a-zA-Z])")
_UNIT_NS = {"ns": 1.0, "us": 1e3, "\u00b5s": 1e3, "ms": 1e6, "s": 1e9}
_NAME = re.compile(r"^(?:test [A-Za-z0-9_:]+ \.\.\. )?(bench_[A-Za-z0-9_]+)\s*:")
EXPECTED_BENCHES = {
"bench_verify_artifact", "bench_naming_transforms",
"bench_parse_line_near_max_addrs", "bench_parse_targets_many_ranges_near_cap",
"bench_targets_to_cidrs_large_range", "bench_view_pipeline",
"bench_render_cells", "bench_health_diagnostics",
"bench_view_pipeline_default_sort", "bench_sort_by_each_column",
"bench_view_pipeline_with_filter", "bench_upsert_cold_start_population",
}
def parse_bench(text: str) -> dict:
@@ -44,12 +33,21 @@ def parse_bench(text: str) -> dict:
continue
durs = _DUR.findall(line)
if not durs:
continue
raise ValueError(f"missing timing for {m.group(1)}")
val, unit = durs[-1]
out[m.group(1)] = float(val) * _UNIT_NS[unit]
elapsed = float(val) * _UNIT_NS[unit]
if m.group(1) in out or not math.isfinite(elapsed) or elapsed <= 0:
raise ValueError(f"duplicate or invalid timing for {m.group(1)}")
out[m.group(1)] = elapsed
return out
def require_suite(results: dict) -> dict:
if results.keys() != EXPECTED_BENCHES:
raise ValueError("microbenchmark results differ from the declared suite")
return results
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--current", required=True, help="captured bench stdout")
@@ -59,25 +57,29 @@ def main() -> int:
try:
with open(args.current, encoding="utf-8") as fh:
current = parse_bench(fh.read())
except OSError as e:
current = require_suite(parse_bench(fh.read()))
except (OSError, ValueError) as e:
print(f"bench-badge: cannot read current results: {e}", file=sys.stderr)
print("NEW")
return 0
if not current:
print("bench-badge: no bench lines parsed from current results", file=sys.stderr)
print("NEW")
return 0
return 1
baseline = {}
try:
with open(args.baseline, encoding="utf-8") as fh:
baseline = {k: float(v) for k, v in json.load(fh).items()}
except (OSError, ValueError):
baseline = {}
baseline = json.load(fh)
except FileNotFoundError:
pass
except (OSError, ValueError) as e:
print(f"bench-badge: invalid baseline: {e}", file=sys.stderr)
return 1
if (not isinstance(baseline, dict)
or not set(baseline).issubset(current)
or any(not isinstance(value, (int, float)) or isinstance(value, bool)
or not math.isfinite(value) or value <= 0
for value in baseline.values())):
print("bench-badge: invalid or retired benchmark in baseline", file=sys.stderr)
return 1
common = [k for k in current if k in baseline and baseline[k] > 0]
common = list(baseline)
if common:
sum_new = sum(current[k] for k in common)
sum_old = sum(baseline[k] for k in common)
+2 -2
View File
@@ -141,8 +141,8 @@ def main() -> None:
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
print(f"Loaded {len(baseline)} prior benchmark measurements")
return
metrics = {"bench_ns_per_op": bench_badge.parse_bench(
Path("bench-current.txt").read_text()), "bench_host": benchmark_host()}
metrics = {"bench_ns_per_op": bench_badge.require_suite(bench_badge.parse_bench(
Path("bench-current.txt").read_text())), "bench_host": benchmark_host()}
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
index = updated_index(index, sha, metrics, timestamp)
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Check that every Toolkit benchmark contributes to the release badge."""
import importlib.util
import re
from pathlib import Path
import unittest
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("bench_badge", ROOT / "ci/bench-badge.py")
assert spec is not None and spec.loader is not None
badge = importlib.util.module_from_spec(spec)
spec.loader.exec_module(badge)
class BenchBadgeTests(unittest.TestCase):
def test_declared_suite_matches_rust_benchmarks(self):
names = set()
for directory in (ROOT / "crates/alkane-gui/src", ROOT / "crates/alkane-artifacts/src"):
for path in directory.rglob("*.rs"):
names.update(re.findall(
r'#\[test\]\s*#\[ignore(?:\s*=\s*"[^"]*")?\]\s*'
r'fn (bench_[A-Za-z0-9_]+)\s*\(', path.read_text()))
self.assertEqual(names, badge.EXPECTED_BENCHES)
def test_incomplete_duplicate_and_malformed_results_fail(self):
lines = [f"test tests::{name} ... {name}: 10ms/op"
for name in sorted(badge.EXPECTED_BENCHES)]
values = badge.require_suite(badge.parse_bench("\n".join(lines)))
self.assertEqual(len(values), 12)
with self.assertRaisesRegex(ValueError, "declared suite"):
badge.require_suite(badge.parse_bench("\n".join(lines[:-1])))
with self.assertRaisesRegex(ValueError, "duplicate"):
badge.parse_bench("\n".join(lines + [lines[0]]))
with self.assertRaisesRegex(ValueError, "missing timing"):
badge.parse_bench(lines[0].replace("10ms/op", "missing"))
if __name__ == "__main__":
unittest.main()