Average benchmark improvements equally in Toolkit badge
build-test-release / release (push) Successful in 16m41s

This commit is contained in:
2026-10-16 06:02:42 -06:00
parent a1f610a393
commit dd0732454e
3 changed files with 41 additions and 19 deletions
+18 -8
View File
@@ -48,6 +48,18 @@ def require_suite(results: dict) -> dict:
return results
def benchmark_delta(current: dict, baseline: dict) -> float | None:
"""Average each comparable benchmark's percentage improvement."""
if not baseline:
return None
changes = [(baseline[name] - current[name]) / baseline[name]
for name in baseline]
delta = math.fsum(changes) / len(changes) * 100
if not math.isfinite(delta):
raise ValueError("benchmark comparison is not finite")
return delta
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--current", required=True, help="captured bench stdout")
@@ -79,14 +91,12 @@ def main() -> int:
print("bench-badge: invalid or retired benchmark in baseline", file=sys.stderr)
return 1
common = list(baseline)
if common:
sum_new = sum(current[k] for k in common)
sum_old = sum(baseline[k] for k in common)
delta = (sum_new / sum_old - 1.0) * 100.0 if sum_old > 0 else 0.0
print(f"{delta:.1f}")
else:
print("NEW")
try:
delta = benchmark_delta(current, baseline)
except ValueError as error:
print(f"bench-badge: {error}", file=sys.stderr)
return 1
print("NEW" if delta is None else f"{delta:.1f}")
if not args.no_write:
os.makedirs(os.path.dirname(args.baseline) or ".", exist_ok=True)
+10 -11
View File
@@ -108,13 +108,13 @@ def loc_color(n: int) -> str:
def bench_color(delta_pct: float) -> str:
# Negative delta = faster than the previous release. A +/-2% dead band absorbs
# the run-to-run jitter of microbenchmarks on a shared CI runner.
if delta_pct <= -2.0:
# Positive delta means faster than the previous release. A +/-2% band
# absorbs run-to-run jitter on the dedicated CI runner.
if delta_pct >= 2.0:
return "brightgreen"
if delta_pct < 2.0:
if delta_pct > -2.0:
return "blue"
if delta_pct < 10.0:
if delta_pct > -10.0:
return "yellow"
return "orange"
@@ -161,9 +161,8 @@ def main() -> int:
ap.add_argument("--coverage", type=float, default=None)
ap.add_argument("--bench-coverage", type=float, default=None)
ap.add_argument("--bench", default=None)
# Weighted-average microbench delta vs the previous release, as a signed
# percentage string (e.g. "-3.2"), or "NEW"/"" when there is no baseline yet
# (round 19 #1). Takes precedence over --bench for the bench badge.
# Mean per-benchmark improvement vs the previous release, as a signed
# percentage string, or "NEW" when there is no baseline.
ap.add_argument("--bench-delta", default=None)
ap.add_argument("--clippy", choices=("clean", "failed"))
args = ap.parse_args()
@@ -197,8 +196,8 @@ def main() -> int:
write("bench-coverage", "bench coverage",
f"{args.bench_coverage:.0f}%" if args.bench_coverage is not None else "pending",
cov_color(args.bench_coverage) if args.bench_coverage is not None else "lightgrey")
# Bench badge: prefer the weighted-average delta vs the previous release; fall
# back to the static label (first run, or when the delta could not be computed).
# Bench badge: prefer the mean per-benchmark improvement vs the previous
# release. Use the static label when no comparison is available.
delta = None
if args.bench_delta not in (None, "", "NEW"):
try:
@@ -206,7 +205,7 @@ def main() -> int:
except ValueError:
delta = None
if delta is not None:
# Negative = faster than the previous release (good).
# Positive = faster than the previous release.
msg = f"{'+' if delta > 0 else ''}{delta:.1f}% vs prev"
write("bench", "bench", msg, bench_color(delta))
elif args.bench_delta == "NEW":
+13
View File
@@ -13,6 +13,10 @@ spec = importlib.util.spec_from_file_location("bench_badge", ROOT / "ci/bench-ba
assert spec is not None and spec.loader is not None
badge = importlib.util.module_from_spec(spec)
spec.loader.exec_module(badge)
badge_spec = importlib.util.spec_from_file_location("gen_badges", ROOT / "ci/gen-badges.py")
assert badge_spec is not None and badge_spec.loader is not None
gen_badges = importlib.util.module_from_spec(badge_spec)
badge_spec.loader.exec_module(gen_badges)
class BenchBadgeTests(unittest.TestCase):
@@ -37,6 +41,15 @@ class BenchBadgeTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "missing timing"):
badge.parse_bench(lines[0].replace("10ms/op", "missing"))
def test_badge_averages_each_benchmark_improvement(self):
current = {"short": 50.0, "long": 900.0}
baseline = {"short": 100.0, "long": 1000.0}
self.assertEqual(badge.benchmark_delta(current, baseline), 30.0)
self.assertAlmostEqual(badge.benchmark_delta(baseline, current), -55.5555556)
self.assertIsNone(badge.benchmark_delta(current, {}))
self.assertEqual(gen_badges.bench_color(30.0), "brightgreen")
self.assertEqual(gen_badges.bench_color(-55.5555556), "orange")
if __name__ == "__main__":
unittest.main()