Calibrate microbenchmarks and measure quality grade
next-build / build (push) Canceled after 0s

This commit is contained in:
2026-10-15 03:17:58 -06:00
parent af49d4905d
commit eb9dcf830b
8 changed files with 117 additions and 10 deletions
+3 -1
View File
@@ -22,6 +22,8 @@ jobs:
run: python3 ci/test-last-attempt-badge.py
- name: Benchmark history checks
run: python3 ci/test-bench-history.py
- name: Metric badge checks
run: python3 ci/test-metrics.py
- name: Release evidence gate checks
run: python3 ci/test-conformance.py
- name: Release publication checks
@@ -37,7 +39,7 @@ jobs:
- name: Microbenchmarks
run: |
set -o pipefail
cargo test --release --locked -- --ignored --nocapture | tee bench-current.txt
cargo test --release --locked -- --ignored --nocapture --test-threads=1 | tee bench-current.txt
- name: Coverage tool
run: |
rustup component add llvm-tools-preview
+4
View File
@@ -113,3 +113,7 @@ not imply that any board image or mining target has passed conformance.
CI and metric badges are served from a private, machine-maintained branch.
The CI badge shows the last completed attempt for this source branch while
newer runs are queued; main CI refreshes measured metrics before conformance.
Microbenchmarks report the median of seven calibrated samples and compare
only with an earlier commit measured on the same CPU class. The code-quality
letter requires warnings-denied Clippy to pass and uses measured test line
coverage: A at 90%, B at 80%, C at 70%, D at 60%, and F below 60%.
+9 -3
View File
@@ -16,6 +16,7 @@ from urllib.error import HTTPError
from urllib.parse import quote
from badge_store import BADGE_BRANCH, publish_file, request_json
from bench_host import identity as benchmark_host
INDEX = "history/index.json"
MAX_INDEX_ENTRIES = 50
@@ -53,13 +54,15 @@ def git_distance(ancestor: str, sha: str) -> int | None:
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
def previous_measurement(index: dict, sha: str, distance=None) -> dict:
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
distance = distance or git_distance
choices = []
for entry in index.get("entries", []):
ancestor = entry.get("sha")
if ancestor == sha or not isinstance(ancestor, str):
continue
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
continue
gap = distance(ancestor, sha)
if gap is not None and gap > 0:
choices.append((gap, entry))
@@ -89,6 +92,9 @@ def render_html(index: dict) -> str:
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-os/commit/{sha}">{sha[:10]}</a></td>',
f'<td>{html.escape(entry["timestamp"])}</td>']
values = valid_benches(entry["metrics"])
host = entry["metrics"].get("bench_host", {})
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
cells.append(f'<td>{html.escape(str(host_label))}</td>')
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
for name in names]
rows.append("<tr>" + "".join(cells) + "</tr>")
@@ -108,7 +114,7 @@ def render_html(index: dict) -> str:
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
"in the private ci-badges branch.</small></p>"
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
f"<th scope=\"col\">UTC</th>{headings}</tr></thead><tbody>{''.join(rows)}"
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>{headings}</tr></thead><tbody>{''.join(rows)}"
"</tbody></table></div></body></html>\n")
@@ -124,7 +130,7 @@ def main() -> None:
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
raise ValueError("unsupported benchmark history index")
if args.action == "prepare":
baseline = previous_measurement(index, sha)
baseline = previous_measurement(index, sha, host=benchmark_host())
Path("badges").mkdir(exist_ok=True)
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
print(f"Loaded {len(baseline)} prior benchmark measurements")
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Identify the CPU class used for comparable CI microbenchmark baselines."""
import os
import platform
from pathlib import Path
def identity() -> dict[str, object]:
model = "unknown"
cpuinfo = Path("/proc/cpuinfo")
if cpuinfo.is_file():
for line in cpuinfo.read_text(errors="replace").splitlines():
if line.startswith(("model name\t", "Hardware\t", "Processor\t")):
model = line.partition(":")[2].strip()[:160]
break
return {"architecture": platform.machine(), "cpu_model": model,
"logical_cpus": os.cpu_count() or 1}
+19 -2
View File
@@ -7,6 +7,7 @@ import json
import pathlib
import re
import tomllib
from bench_host import identity as benchmark_host
def coverage(path):
@@ -33,6 +34,19 @@ def compact(number):
return number + units[index]
def quality_grade(line_coverage):
# Main CI reaches this publisher only after warnings-denied Clippy passes.
if line_coverage >= 90:
return "A"
if line_coverage >= 80:
return "B"
if line_coverage >= 70:
return "C"
if line_coverage >= 60:
return "D"
return "F"
def svg(label, value, color):
label, value = html.escape(label), html.escape(value)
lw, vw = 12 + 7 * len(label), 12 + 7 * len(value)
@@ -74,8 +88,9 @@ def main():
metrics = {"tests": tests, "line_coverage_pct": round(line_coverage, 1),
"bench_coverage_pct": round(bench_coverage, 1),
"bench_ns_per_op": current, "bench_improvement_pct": delta,
"bench_host": benchmark_host(),
"lines_of_code": loc, "dependencies": dependencies,
"code_quality": "A", "license": "MIT"}
"code_quality": quality_grade(line_coverage), "license": "MIT"}
pathlib.Path("badges").mkdir(exist_ok=True)
pathlib.Path("badges/metrics.json").write_text(json.dumps(metrics, indent=2) + "\n")
badges = {
@@ -85,7 +100,9 @@ def main():
"bench": ("bench", "baseline set" if delta is None else f"{delta:+.1f}% vs prev", "#007ec6"),
"loc": ("lines of code", compact(loc), "#007ec6"),
"dependencies": ("dependencies", str(dependencies), "#007ec6"),
"quality": ("code quality", "A", "#4c1"),
"quality": ("code quality", metrics["code_quality"],
{"A": "#4c1", "B": "#97ca00", "C": "#dfb317",
"D": "#fe7d37", "F": "#e05d44"}[metrics["code_quality"]]),
"license": ("license", "MIT", "#007ec6"),
}
for name, (label, value, color) in badges.items():
+13 -1
View File
@@ -33,9 +33,21 @@ class HistoryTests(unittest.TestCase):
with self.assertRaises(ValueError):
updated_index({}, "a" * 40, {"bench_ns_per_op": {"hash": -1}}, "now")
def test_baseline_must_match_cpu_class(self):
first = "a" * 40
host = {"architecture": "x86_64", "cpu_model": "A", "logical_cpus": 8}
other = {"architecture": "x86_64", "cpu_model": "B", "logical_cpus": 8}
index = {"entries": [{"sha": first, "metrics": {
"bench_ns_per_op": {"hash": 10.0}, "bench_host": host}}]}
distance = lambda _ancestor, _sha: 1
self.assertEqual(previous_measurement(index, "b" * 40, distance, host),
{"hash": 10.0})
self.assertEqual(previous_measurement(index, "b" * 40, distance, other), {})
def test_site_escapes_history_content(self):
entry = {"sha": "a" * 40, "timestamp": "<script>alert(1)</script>",
"metrics": {"bench_ns_per_op": {"hash": 10.0}}}
"metrics": {"bench_ns_per_op": {"hash": 10.0},
"bench_host": {"cpu_model": "<script>cpu</script>"}}}
page = render_html({"entries": [entry]})
self.assertIn("&lt;script&gt;", page)
self.assertNotIn("<script>", page)
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Regression checks for measured benchmark and quality badges."""
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parent))
from metrics import bench_results, compact, quality_grade
class MetricTests(unittest.TestCase):
def test_quality_grade_depends_on_measured_coverage(self):
self.assertEqual(quality_grade(92), "A")
self.assertEqual(quality_grade(88), "B")
self.assertEqual(quality_grade(72), "C")
self.assertEqual(quality_grade(65), "D")
self.assertEqual(quality_grade(40), "F")
def test_benchmark_parser_and_short_si_badge(self):
with TemporaryDirectory() as directory:
path = Path(directory) / "bench.txt"
path.write_text("BENCH sha 42.125 ns/op\n")
self.assertEqual(bench_results(path), {"sha": 42.125})
self.assertEqual(compact(1234567), "1.23M")
if __name__ == "__main__":
unittest.main()
+19 -3
View File
@@ -12,14 +12,30 @@ use alkane_os::stratum_wire::authorize_request;
use alkane_os::stratum_work::Sha256Job;
use alkane_os::{power_decision, target_matrix, HardwareBackend};
use std::hint::black_box;
use std::time::Instant;
use std::time::{Duration, Instant};
fn measure(name: &str, iterations: u64, mut f: impl FnMut()) {
fn sample(iterations: u64, f: &mut impl FnMut()) -> Duration {
let started = Instant::now();
for _ in 0..iterations {
f();
}
let ns = started.elapsed().as_nanos() as f64 / iterations as f64;
started.elapsed()
}
fn measure(name: &str, minimum_iterations: u64, mut f: impl FnMut()) {
for _ in 0..minimum_iterations.min(1000) {
f();
}
let mut iterations = minimum_iterations.max(1);
while sample(iterations, &mut f) < Duration::from_millis(20) && iterations < 100_000_000 {
iterations = (iterations * 2).min(100_000_000);
}
let mut measurements = [0_f64; 7];
for value in &mut measurements {
*value = sample(iterations, &mut f).as_nanos() as f64 / iterations as f64;
}
measurements.sort_by(f64::total_cmp);
let ns = measurements[measurements.len() / 2];
println!("BENCH {name} {ns:.3} ns/op");
}