Compare toolkit benchmarks on matching CPU classes

This commit is contained in:
2026-10-15 03:37:30 -06:00
parent 03c19ef7c6
commit 9879a0eb25
6 changed files with 46 additions and 7 deletions
+1 -1
View File
@@ -92,7 +92,7 @@ jobs:
# #1) while keeping the step a hard gate: pipefail preserves cargo's exit
# code (each bench asserts its own absolute ceiling) even through tee.
set -o pipefail
cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ \
cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ --test-threads=1 \
| tee bench-current.txt
- name: Record private benchmark history
+1 -1
View File
@@ -55,7 +55,7 @@ jobs:
cargo test -p alkane-artifacts -p alkane-flash -p alkane-cli --offline
- name: Microbenchmarks
run: cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_
run: cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ --test-threads=1
- name: Headless UI self-test
run: |
+3
View File
@@ -15,6 +15,9 @@ runs are queued. Badge SVGs are stored in a private, machine-maintained branch.
The code-quality letter requires warnings-denied Clippy and measured test plus
UI self-test line coverage: A at 90%, B at 80%, C at 70%, D at 60%, and F
below 60%. A missing coverage measurement displays pending.
The benchmark delta compares an earlier commit only when its CPU class
matches the current dedicated runner; otherwise it establishes a new
baseline. Main CI runs microbenchmarks serially to avoid test contention.
A cross-platform desktop toolkit for discovering, monitoring, and re-flashing
ASIC miners. It scans a fleet over the network, shows live health and hashrate
+10 -4
View File
@@ -17,6 +17,7 @@ from urllib.error import HTTPError
from urllib.parse import quote
from badge_store import BADGE_BRANCH, publish_file, request_json
from bench_host import identity as benchmark_host
bench_spec = importlib.util.spec_from_file_location(
"bench_badge", Path(__file__).with_name("bench-badge.py"))
@@ -59,13 +60,15 @@ def git_distance(ancestor: str, sha: str) -> int | None:
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
def previous_measurement(index: dict, sha: str, distance=None) -> dict:
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
distance = distance or git_distance
choices = []
for entry in index.get("entries", []):
ancestor = entry.get("sha")
if ancestor == sha or not isinstance(ancestor, str):
continue
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
continue
gap = distance(ancestor, sha)
if gap is not None and gap > 0:
choices.append((gap, entry))
@@ -95,6 +98,9 @@ def render_html(index: dict) -> str:
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-toolkit/commit/{sha}">{sha[:10]}</a></td>',
f'<td>{html.escape(entry["timestamp"])}</td>']
values = valid_benches(entry["metrics"])
host = entry["metrics"].get("bench_host", {})
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
cells.append(f'<td>{html.escape(str(host_label))}</td>')
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
for name in names]
rows.append("<tr>" + "".join(cells) + "</tr>")
@@ -114,7 +120,7 @@ def render_html(index: dict) -> str:
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
"in the private ci-badges branch.</small></p>"
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
f"<th scope=\"col\">UTC</th>{headings}</tr></thead><tbody>{''.join(rows)}"
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>{headings}</tr></thead><tbody>{''.join(rows)}"
"</tbody></table></div></body></html>\n")
@@ -130,13 +136,13 @@ def main() -> None:
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
raise ValueError("unsupported benchmark history index")
if args.action == "prepare":
baseline = previous_measurement(index, sha)
baseline = previous_measurement(index, sha, host=benchmark_host())
Path("badges").mkdir(exist_ok=True)
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
print(f"Loaded {len(baseline)} prior benchmark measurements")
return
metrics = {"bench_ns_per_op": bench_badge.parse_bench(
Path("bench-current.txt").read_text())}
Path("bench-current.txt").read_text()), "bench_host": benchmark_host()}
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
index = updated_index(index, sha, metrics, timestamp)
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Identify the CPU class used for comparable CI microbenchmark baselines."""
import os
import platform
from pathlib import Path
def identity() -> dict[str, object]:
model = "unknown"
cpuinfo = Path("/proc/cpuinfo")
if cpuinfo.is_file():
for line in cpuinfo.read_text(errors="replace").splitlines():
if line.startswith(("model name\t", "Hardware\t", "Processor\t")):
model = line.partition(":")[2].strip()[:160]
break
return {"architecture": platform.machine(), "cpu_model": model,
"logical_cpus": os.cpu_count() or 1}
+12 -1
View File
@@ -33,9 +33,20 @@ class HistoryTests(unittest.TestCase):
with self.assertRaises(ValueError):
updated_index({}, "a" * 40, {"bench_ns_per_op": {"hash": -1}}, "now")
def test_baseline_must_match_cpu_class(self):
host = {"architecture": "x86_64", "cpu_model": "A", "logical_cpus": 8}
other = {"architecture": "x86_64", "cpu_model": "B", "logical_cpus": 8}
index = {"entries": [{"sha": "a" * 40, "metrics": {
"bench_ns_per_op": {"hash": 10.0}, "bench_host": host}}]}
distance = lambda _ancestor, _sha: 1
self.assertEqual(previous_measurement(index, "b" * 40, distance, host),
{"hash": 10.0})
self.assertEqual(previous_measurement(index, "b" * 40, distance, other), {})
def test_site_escapes_history_content(self):
entry = {"sha": "a" * 40, "timestamp": "<script>alert(1)</script>",
"metrics": {"bench_ns_per_op": {"hash": 10.0}}}
"metrics": {"bench_ns_per_op": {"hash": 10.0},
"bench_host": {"cpu_model": "<script>cpu</script>"}}}
page = render_html({"entries": [entry]})
self.assertIn("&lt;script&gt;", page)
self.assertNotIn("<script>", page)