Require complete Toolkit benchmarks before release
build-test-release / release (push) Successful in 16m41s
build-test-release / release (push) Successful in 16m41s
This commit is contained in:
@@ -38,6 +38,9 @@ jobs:
|
||||
- name: Benchmark history checks
|
||||
run: python3 ci/test-bench-history.py
|
||||
|
||||
- name: Benchmark suite checks
|
||||
run: python3 ci/test-bench-badge.py
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
@@ -89,12 +92,12 @@ jobs:
|
||||
|
||||
- name: Microbenchmarks
|
||||
run: |
|
||||
# Capture the bench stdout for the weighted-average delta badge (round 19
|
||||
# #1) while keeping the step a hard gate: pipefail preserves cargo's exit
|
||||
# code (each bench asserts its own absolute ceiling) even through tee.
|
||||
# Require every declared benchmark before publishing history or a release.
|
||||
set -o pipefail
|
||||
cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ --test-threads=1 \
|
||||
| tee bench-current.txt
|
||||
python3 ci/bench-badge.py --current bench-current.txt \
|
||||
--baseline badges/bench-baseline.json --no-write
|
||||
|
||||
- name: Record private benchmark history
|
||||
env:
|
||||
|
||||
+40
-38
@@ -1,27 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute the weighted-average microbenchmark delta versus the previous release.
|
||||
|
||||
Reads the captured stdout of `cargo test ... --ignored --nocapture bench_` (whose
|
||||
timings are Rust `Duration` Debug strings: "12.3ms", "512us"/"512µs", "840ns",
|
||||
"1.2s"), compares each bench to a committed baseline of the previous release's
|
||||
numbers, and prints the time-weighted average change as a signed percentage:
|
||||
|
||||
(sum(current) / sum(baseline) - 1) * 100 over benches present in both
|
||||
|
||||
Weighting by duration means the slow benches dominate, so the number tracks
|
||||
overall wall-time change rather than giving a tiny bench equal say. Then the
|
||||
baseline is rewritten with the current numbers, so the next release compares
|
||||
against this one ("the previous commit").
|
||||
|
||||
Usage:
|
||||
bench-badge.py --current bench-current.txt --baseline badges/bench-baseline.json
|
||||
|
||||
Prints the delta as a plain number (e.g. "-3.2", or "NEW" when there is no
|
||||
usable baseline) to stdout. The baseline JSON lives under badges/ so committing
|
||||
it back does not trip compute-version.sh's path-based release bump.
|
||||
"""
|
||||
"""Check the Toolkit benchmark suite and compare it with the prior release."""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
@@ -29,9 +10,17 @@ import sys
|
||||
# A Duration Debug token: a number immediately followed by a time unit, with the
|
||||
# multi-char units listed before the bare "s" so alternation prefers them. The
|
||||
# trailing (?![a-zA-Z]) stops "s" in words like "rows"/"cidrs" from matching.
|
||||
_DUR = re.compile(r"([0-9]+(?:\.[0-9]+)?)\s*(ns|µs|us|ms|s)(?![a-zA-Z])")
|
||||
_UNIT_NS = {"ns": 1.0, "us": 1e3, "µs": 1e3, "ms": 1e6, "s": 1e9}
|
||||
_DUR = re.compile(r"([0-9]+(?:\.[0-9]+)?)\s*(ns|\u00b5s|us|ms|s)(?![a-zA-Z])")
|
||||
_UNIT_NS = {"ns": 1.0, "us": 1e3, "\u00b5s": 1e3, "ms": 1e6, "s": 1e9}
|
||||
_NAME = re.compile(r"^(?:test [A-Za-z0-9_:]+ \.\.\. )?(bench_[A-Za-z0-9_]+)\s*:")
|
||||
EXPECTED_BENCHES = {
|
||||
"bench_verify_artifact", "bench_naming_transforms",
|
||||
"bench_parse_line_near_max_addrs", "bench_parse_targets_many_ranges_near_cap",
|
||||
"bench_targets_to_cidrs_large_range", "bench_view_pipeline",
|
||||
"bench_render_cells", "bench_health_diagnostics",
|
||||
"bench_view_pipeline_default_sort", "bench_sort_by_each_column",
|
||||
"bench_view_pipeline_with_filter", "bench_upsert_cold_start_population",
|
||||
}
|
||||
|
||||
|
||||
def parse_bench(text: str) -> dict:
|
||||
@@ -44,12 +33,21 @@ def parse_bench(text: str) -> dict:
|
||||
continue
|
||||
durs = _DUR.findall(line)
|
||||
if not durs:
|
||||
continue
|
||||
raise ValueError(f"missing timing for {m.group(1)}")
|
||||
val, unit = durs[-1]
|
||||
out[m.group(1)] = float(val) * _UNIT_NS[unit]
|
||||
elapsed = float(val) * _UNIT_NS[unit]
|
||||
if m.group(1) in out or not math.isfinite(elapsed) or elapsed <= 0:
|
||||
raise ValueError(f"duplicate or invalid timing for {m.group(1)}")
|
||||
out[m.group(1)] = elapsed
|
||||
return out
|
||||
|
||||
|
||||
def require_suite(results: dict) -> dict:
|
||||
if results.keys() != EXPECTED_BENCHES:
|
||||
raise ValueError("microbenchmark results differ from the declared suite")
|
||||
return results
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--current", required=True, help="captured bench stdout")
|
||||
@@ -59,25 +57,29 @@ def main() -> int:
|
||||
|
||||
try:
|
||||
with open(args.current, encoding="utf-8") as fh:
|
||||
current = parse_bench(fh.read())
|
||||
except OSError as e:
|
||||
current = require_suite(parse_bench(fh.read()))
|
||||
except (OSError, ValueError) as e:
|
||||
print(f"bench-badge: cannot read current results: {e}", file=sys.stderr)
|
||||
print("NEW")
|
||||
return 0
|
||||
|
||||
if not current:
|
||||
print("bench-badge: no bench lines parsed from current results", file=sys.stderr)
|
||||
print("NEW")
|
||||
return 0
|
||||
return 1
|
||||
|
||||
baseline = {}
|
||||
try:
|
||||
with open(args.baseline, encoding="utf-8") as fh:
|
||||
baseline = {k: float(v) for k, v in json.load(fh).items()}
|
||||
except (OSError, ValueError):
|
||||
baseline = {}
|
||||
baseline = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except (OSError, ValueError) as e:
|
||||
print(f"bench-badge: invalid baseline: {e}", file=sys.stderr)
|
||||
return 1
|
||||
if (not isinstance(baseline, dict)
|
||||
or not set(baseline).issubset(current)
|
||||
or any(not isinstance(value, (int, float)) or isinstance(value, bool)
|
||||
or not math.isfinite(value) or value <= 0
|
||||
for value in baseline.values())):
|
||||
print("bench-badge: invalid or retired benchmark in baseline", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
common = [k for k in current if k in baseline and baseline[k] > 0]
|
||||
common = list(baseline)
|
||||
if common:
|
||||
sum_new = sum(current[k] for k in common)
|
||||
sum_old = sum(baseline[k] for k in common)
|
||||
|
||||
+2
-2
@@ -141,8 +141,8 @@ def main() -> None:
|
||||
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
|
||||
print(f"Loaded {len(baseline)} prior benchmark measurements")
|
||||
return
|
||||
metrics = {"bench_ns_per_op": bench_badge.parse_bench(
|
||||
Path("bench-current.txt").read_text()), "bench_host": benchmark_host()}
|
||||
metrics = {"bench_ns_per_op": bench_badge.require_suite(bench_badge.parse_bench(
|
||||
Path("bench-current.txt").read_text())), "bench_host": benchmark_host()}
|
||||
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
index = updated_index(index, sha, metrics, timestamp)
|
||||
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Check that every Toolkit benchmark contributes to the release badge."""
|
||||
|
||||
import importlib.util
|
||||
import re
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location("bench_badge", ROOT / "ci/bench-badge.py")
|
||||
assert spec is not None and spec.loader is not None
|
||||
badge = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(badge)
|
||||
|
||||
|
||||
class BenchBadgeTests(unittest.TestCase):
|
||||
def test_declared_suite_matches_rust_benchmarks(self):
|
||||
names = set()
|
||||
for directory in (ROOT / "crates/alkane-gui/src", ROOT / "crates/alkane-artifacts/src"):
|
||||
for path in directory.rglob("*.rs"):
|
||||
names.update(re.findall(
|
||||
r'#\[test\]\s*#\[ignore(?:\s*=\s*"[^"]*")?\]\s*'
|
||||
r'fn (bench_[A-Za-z0-9_]+)\s*\(', path.read_text()))
|
||||
self.assertEqual(names, badge.EXPECTED_BENCHES)
|
||||
|
||||
def test_incomplete_duplicate_and_malformed_results_fail(self):
|
||||
lines = [f"test tests::{name} ... {name}: 10ms/op"
|
||||
for name in sorted(badge.EXPECTED_BENCHES)]
|
||||
values = badge.require_suite(badge.parse_bench("\n".join(lines)))
|
||||
self.assertEqual(len(values), 12)
|
||||
with self.assertRaisesRegex(ValueError, "declared suite"):
|
||||
badge.require_suite(badge.parse_bench("\n".join(lines[:-1])))
|
||||
with self.assertRaisesRegex(ValueError, "duplicate"):
|
||||
badge.parse_bench("\n".join(lines + [lines[0]]))
|
||||
with self.assertRaisesRegex(ValueError, "missing timing"):
|
||||
badge.parse_bench(lines[0].replace("10ms/op", "missing"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user