Compare toolkit benchmarks on matching CPU classes
This commit is contained in:
@@ -92,7 +92,7 @@ jobs:
|
||||
# #1) while keeping the step a hard gate: pipefail preserves cargo's exit
|
||||
# code (each bench asserts its own absolute ceiling) even through tee.
|
||||
set -o pipefail
|
||||
cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ \
|
||||
cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ --test-threads=1 \
|
||||
| tee bench-current.txt
|
||||
|
||||
- name: Record private benchmark history
|
||||
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
cargo test -p alkane-artifacts -p alkane-flash -p alkane-cli --offline
|
||||
|
||||
- name: Microbenchmarks
|
||||
run: cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_
|
||||
run: cargo test -p alkane-gui -p alkane-artifacts --release --offline -- --ignored --nocapture bench_ --test-threads=1
|
||||
|
||||
- name: Headless UI self-test
|
||||
run: |
|
||||
|
||||
@@ -15,6 +15,9 @@ runs are queued. Badge SVGs are stored in a private, machine-maintained branch.
|
||||
The code-quality letter requires warnings-denied Clippy and measured test plus
|
||||
UI self-test line coverage: A at 90%, B at 80%, C at 70%, D at 60%, and F
|
||||
below 60%. A missing coverage measurement displays pending.
|
||||
The benchmark delta compares an earlier commit only when its CPU class
|
||||
matches the current dedicated runner; otherwise it establishes a new
|
||||
baseline. Main CI runs microbenchmarks serially to avoid test contention.
|
||||
|
||||
A cross-platform desktop toolkit for discovering, monitoring, and re-flashing
|
||||
ASIC miners. It scans a fleet over the network, shows live health and hashrate
|
||||
|
||||
+10
-4
@@ -17,6 +17,7 @@ from urllib.error import HTTPError
|
||||
from urllib.parse import quote
|
||||
|
||||
from badge_store import BADGE_BRANCH, publish_file, request_json
|
||||
from bench_host import identity as benchmark_host
|
||||
|
||||
bench_spec = importlib.util.spec_from_file_location(
|
||||
"bench_badge", Path(__file__).with_name("bench-badge.py"))
|
||||
@@ -59,13 +60,15 @@ def git_distance(ancestor: str, sha: str) -> int | None:
|
||||
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
|
||||
|
||||
|
||||
def previous_measurement(index: dict, sha: str, distance=None) -> dict:
|
||||
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
|
||||
distance = distance or git_distance
|
||||
choices = []
|
||||
for entry in index.get("entries", []):
|
||||
ancestor = entry.get("sha")
|
||||
if ancestor == sha or not isinstance(ancestor, str):
|
||||
continue
|
||||
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
|
||||
continue
|
||||
gap = distance(ancestor, sha)
|
||||
if gap is not None and gap > 0:
|
||||
choices.append((gap, entry))
|
||||
@@ -95,6 +98,9 @@ def render_html(index: dict) -> str:
|
||||
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-toolkit/commit/{sha}">{sha[:10]}</a></td>',
|
||||
f'<td>{html.escape(entry["timestamp"])}</td>']
|
||||
values = valid_benches(entry["metrics"])
|
||||
host = entry["metrics"].get("bench_host", {})
|
||||
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
|
||||
cells.append(f'<td>{html.escape(str(host_label))}</td>')
|
||||
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
|
||||
for name in names]
|
||||
rows.append("<tr>" + "".join(cells) + "</tr>")
|
||||
@@ -114,7 +120,7 @@ def render_html(index: dict) -> str:
|
||||
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
|
||||
"in the private ci-badges branch.</small></p>"
|
||||
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
|
||||
f"<th scope=\"col\">UTC</th>{headings}</tr></thead><tbody>{''.join(rows)}"
|
||||
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>{headings}</tr></thead><tbody>{''.join(rows)}"
|
||||
"</tbody></table></div></body></html>\n")
|
||||
|
||||
|
||||
@@ -130,13 +136,13 @@ def main() -> None:
|
||||
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
|
||||
raise ValueError("unsupported benchmark history index")
|
||||
if args.action == "prepare":
|
||||
baseline = previous_measurement(index, sha)
|
||||
baseline = previous_measurement(index, sha, host=benchmark_host())
|
||||
Path("badges").mkdir(exist_ok=True)
|
||||
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
|
||||
print(f"Loaded {len(baseline)} prior benchmark measurements")
|
||||
return
|
||||
metrics = {"bench_ns_per_op": bench_badge.parse_bench(
|
||||
Path("bench-current.txt").read_text())}
|
||||
Path("bench-current.txt").read_text()), "bench_host": benchmark_host()}
|
||||
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
index = updated_index(index, sha, metrics, timestamp)
|
||||
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Identify the CPU class used for comparable CI microbenchmark baselines."""
|
||||
|
||||
import os
|
||||
import platform
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def identity() -> dict[str, object]:
|
||||
model = "unknown"
|
||||
cpuinfo = Path("/proc/cpuinfo")
|
||||
if cpuinfo.is_file():
|
||||
for line in cpuinfo.read_text(errors="replace").splitlines():
|
||||
if line.startswith(("model name\t", "Hardware\t", "Processor\t")):
|
||||
model = line.partition(":")[2].strip()[:160]
|
||||
break
|
||||
return {"architecture": platform.machine(), "cpu_model": model,
|
||||
"logical_cpus": os.cpu_count() or 1}
|
||||
@@ -33,9 +33,20 @@ class HistoryTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
updated_index({}, "a" * 40, {"bench_ns_per_op": {"hash": -1}}, "now")
|
||||
|
||||
def test_baseline_must_match_cpu_class(self):
|
||||
host = {"architecture": "x86_64", "cpu_model": "A", "logical_cpus": 8}
|
||||
other = {"architecture": "x86_64", "cpu_model": "B", "logical_cpus": 8}
|
||||
index = {"entries": [{"sha": "a" * 40, "metrics": {
|
||||
"bench_ns_per_op": {"hash": 10.0}, "bench_host": host}}]}
|
||||
distance = lambda _ancestor, _sha: 1
|
||||
self.assertEqual(previous_measurement(index, "b" * 40, distance, host),
|
||||
{"hash": 10.0})
|
||||
self.assertEqual(previous_measurement(index, "b" * 40, distance, other), {})
|
||||
|
||||
def test_site_escapes_history_content(self):
|
||||
entry = {"sha": "a" * 40, "timestamp": "<script>alert(1)</script>",
|
||||
"metrics": {"bench_ns_per_op": {"hash": 10.0}}}
|
||||
"metrics": {"bench_ns_per_op": {"hash": 10.0},
|
||||
"bench_host": {"cpu_model": "<script>cpu</script>"}}}
|
||||
page = render_html({"entries": [entry]})
|
||||
self.assertIn("<script>", page)
|
||||
self.assertNotIn("<script>", page)
|
||||
|
||||
Reference in New Issue
Block a user