This commit is contained in:
@@ -22,6 +22,8 @@ jobs:
|
||||
run: python3 ci/test-last-attempt-badge.py
|
||||
- name: Benchmark history checks
|
||||
run: python3 ci/test-bench-history.py
|
||||
- name: Metric badge checks
|
||||
run: python3 ci/test-metrics.py
|
||||
- name: Release evidence gate checks
|
||||
run: python3 ci/test-conformance.py
|
||||
- name: Release publication checks
|
||||
@@ -37,7 +39,7 @@ jobs:
|
||||
- name: Microbenchmarks
|
||||
run: |
|
||||
set -o pipefail
|
||||
cargo test --release --locked -- --ignored --nocapture | tee bench-current.txt
|
||||
cargo test --release --locked -- --ignored --nocapture --test-threads=1 | tee bench-current.txt
|
||||
- name: Coverage tool
|
||||
run: |
|
||||
rustup component add llvm-tools-preview
|
||||
|
||||
@@ -113,3 +113,7 @@ not imply that any board image or mining target has passed conformance.
|
||||
CI and metric badges are served from a private, machine-maintained branch.
|
||||
The CI badge shows the last completed attempt for this source branch while
|
||||
newer runs are queued; main CI refreshes measured metrics before conformance.
|
||||
Microbenchmarks report the median of seven calibrated samples and compare
|
||||
only with an earlier commit measured on the same CPU class. The code-quality
|
||||
letter requires warnings-denied Clippy to pass and uses measured test line
|
||||
coverage: A at 90%, B at 80%, C at 70%, D at 60%, and F below 60%.
|
||||
|
||||
+9
-3
@@ -16,6 +16,7 @@ from urllib.error import HTTPError
|
||||
from urllib.parse import quote
|
||||
|
||||
from badge_store import BADGE_BRANCH, publish_file, request_json
|
||||
from bench_host import identity as benchmark_host
|
||||
|
||||
INDEX = "history/index.json"
|
||||
MAX_INDEX_ENTRIES = 50
|
||||
@@ -53,13 +54,15 @@ def git_distance(ancestor: str, sha: str) -> int | None:
|
||||
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
|
||||
|
||||
|
||||
def previous_measurement(index: dict, sha: str, distance=None) -> dict:
|
||||
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
|
||||
distance = distance or git_distance
|
||||
choices = []
|
||||
for entry in index.get("entries", []):
|
||||
ancestor = entry.get("sha")
|
||||
if ancestor == sha or not isinstance(ancestor, str):
|
||||
continue
|
||||
if host is not None and entry.get("metrics", {}).get("bench_host") != host:
|
||||
continue
|
||||
gap = distance(ancestor, sha)
|
||||
if gap is not None and gap > 0:
|
||||
choices.append((gap, entry))
|
||||
@@ -89,6 +92,9 @@ def render_html(index: dict) -> str:
|
||||
cells = [f'<td><a href="https://gitea.bluefla.re/blueflare/alkane-os/commit/{sha}">{sha[:10]}</a></td>',
|
||||
f'<td>{html.escape(entry["timestamp"])}</td>']
|
||||
values = valid_benches(entry["metrics"])
|
||||
host = entry["metrics"].get("bench_host", {})
|
||||
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
|
||||
cells.append(f'<td>{html.escape(str(host_label))}</td>')
|
||||
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
|
||||
for name in names]
|
||||
rows.append("<tr>" + "".join(cells) + "</tr>")
|
||||
@@ -108,7 +114,7 @@ def render_html(index: dict) -> str:
|
||||
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
|
||||
"in the private ci-badges branch.</small></p>"
|
||||
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
|
||||
f"<th scope=\"col\">UTC</th>{headings}</tr></thead><tbody>{''.join(rows)}"
|
||||
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>{headings}</tr></thead><tbody>{''.join(rows)}"
|
||||
"</tbody></table></div></body></html>\n")
|
||||
|
||||
|
||||
@@ -124,7 +130,7 @@ def main() -> None:
|
||||
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
|
||||
raise ValueError("unsupported benchmark history index")
|
||||
if args.action == "prepare":
|
||||
baseline = previous_measurement(index, sha)
|
||||
baseline = previous_measurement(index, sha, host=benchmark_host())
|
||||
Path("badges").mkdir(exist_ok=True)
|
||||
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
|
||||
print(f"Loaded {len(baseline)} prior benchmark measurements")
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Identify the CPU class used for comparable CI microbenchmark baselines."""
|
||||
|
||||
import os
|
||||
import platform
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def identity() -> dict[str, object]:
|
||||
model = "unknown"
|
||||
cpuinfo = Path("/proc/cpuinfo")
|
||||
if cpuinfo.is_file():
|
||||
for line in cpuinfo.read_text(errors="replace").splitlines():
|
||||
if line.startswith(("model name\t", "Hardware\t", "Processor\t")):
|
||||
model = line.partition(":")[2].strip()[:160]
|
||||
break
|
||||
return {"architecture": platform.machine(), "cpu_model": model,
|
||||
"logical_cpus": os.cpu_count() or 1}
|
||||
+19
-2
@@ -7,6 +7,7 @@ import json
|
||||
import pathlib
|
||||
import re
|
||||
import tomllib
|
||||
from bench_host import identity as benchmark_host
|
||||
|
||||
|
||||
def coverage(path):
|
||||
@@ -33,6 +34,19 @@ def compact(number):
|
||||
return number + units[index]
|
||||
|
||||
|
||||
def quality_grade(line_coverage):
|
||||
# Main CI reaches this publisher only after warnings-denied Clippy passes.
|
||||
if line_coverage >= 90:
|
||||
return "A"
|
||||
if line_coverage >= 80:
|
||||
return "B"
|
||||
if line_coverage >= 70:
|
||||
return "C"
|
||||
if line_coverage >= 60:
|
||||
return "D"
|
||||
return "F"
|
||||
|
||||
|
||||
def svg(label, value, color):
|
||||
label, value = html.escape(label), html.escape(value)
|
||||
lw, vw = 12 + 7 * len(label), 12 + 7 * len(value)
|
||||
@@ -74,8 +88,9 @@ def main():
|
||||
metrics = {"tests": tests, "line_coverage_pct": round(line_coverage, 1),
|
||||
"bench_coverage_pct": round(bench_coverage, 1),
|
||||
"bench_ns_per_op": current, "bench_improvement_pct": delta,
|
||||
"bench_host": benchmark_host(),
|
||||
"lines_of_code": loc, "dependencies": dependencies,
|
||||
"code_quality": "A", "license": "MIT"}
|
||||
"code_quality": quality_grade(line_coverage), "license": "MIT"}
|
||||
pathlib.Path("badges").mkdir(exist_ok=True)
|
||||
pathlib.Path("badges/metrics.json").write_text(json.dumps(metrics, indent=2) + "\n")
|
||||
badges = {
|
||||
@@ -85,7 +100,9 @@ def main():
|
||||
"bench": ("bench", "baseline set" if delta is None else f"{delta:+.1f}% vs prev", "#007ec6"),
|
||||
"loc": ("lines of code", compact(loc), "#007ec6"),
|
||||
"dependencies": ("dependencies", str(dependencies), "#007ec6"),
|
||||
"quality": ("code quality", "A", "#4c1"),
|
||||
"quality": ("code quality", metrics["code_quality"],
|
||||
{"A": "#4c1", "B": "#97ca00", "C": "#dfb317",
|
||||
"D": "#fe7d37", "F": "#e05d44"}[metrics["code_quality"]]),
|
||||
"license": ("license", "MIT", "#007ec6"),
|
||||
}
|
||||
for name, (label, value, color) in badges.items():
|
||||
|
||||
@@ -33,9 +33,21 @@ class HistoryTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
updated_index({}, "a" * 40, {"bench_ns_per_op": {"hash": -1}}, "now")
|
||||
|
||||
def test_baseline_must_match_cpu_class(self):
|
||||
first = "a" * 40
|
||||
host = {"architecture": "x86_64", "cpu_model": "A", "logical_cpus": 8}
|
||||
other = {"architecture": "x86_64", "cpu_model": "B", "logical_cpus": 8}
|
||||
index = {"entries": [{"sha": first, "metrics": {
|
||||
"bench_ns_per_op": {"hash": 10.0}, "bench_host": host}}]}
|
||||
distance = lambda _ancestor, _sha: 1
|
||||
self.assertEqual(previous_measurement(index, "b" * 40, distance, host),
|
||||
{"hash": 10.0})
|
||||
self.assertEqual(previous_measurement(index, "b" * 40, distance, other), {})
|
||||
|
||||
def test_site_escapes_history_content(self):
|
||||
entry = {"sha": "a" * 40, "timestamp": "<script>alert(1)</script>",
|
||||
"metrics": {"bench_ns_per_op": {"hash": 10.0}}}
|
||||
"metrics": {"bench_ns_per_op": {"hash": 10.0},
|
||||
"bench_host": {"cpu_model": "<script>cpu</script>"}}}
|
||||
page = render_html({"entries": [entry]})
|
||||
self.assertIn("<script>", page)
|
||||
self.assertNotIn("<script>", page)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Regression checks for measured benchmark and quality badges."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from metrics import bench_results, compact, quality_grade
|
||||
|
||||
|
||||
class MetricTests(unittest.TestCase):
|
||||
def test_quality_grade_depends_on_measured_coverage(self):
|
||||
self.assertEqual(quality_grade(92), "A")
|
||||
self.assertEqual(quality_grade(88), "B")
|
||||
self.assertEqual(quality_grade(72), "C")
|
||||
self.assertEqual(quality_grade(65), "D")
|
||||
self.assertEqual(quality_grade(40), "F")
|
||||
|
||||
def test_benchmark_parser_and_short_si_badge(self):
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "bench.txt"
|
||||
path.write_text("BENCH sha 42.125 ns/op\n")
|
||||
self.assertEqual(bench_results(path), {"sha": 42.125})
|
||||
self.assertEqual(compact(1234567), "1.23M")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+19
-3
@@ -12,14 +12,30 @@ use alkane_os::stratum_wire::authorize_request;
|
||||
use alkane_os::stratum_work::Sha256Job;
|
||||
use alkane_os::{power_decision, target_matrix, HardwareBackend};
|
||||
use std::hint::black_box;
|
||||
use std::time::Instant;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
fn measure(name: &str, iterations: u64, mut f: impl FnMut()) {
|
||||
fn sample(iterations: u64, f: &mut impl FnMut()) -> Duration {
|
||||
let started = Instant::now();
|
||||
for _ in 0..iterations {
|
||||
f();
|
||||
}
|
||||
let ns = started.elapsed().as_nanos() as f64 / iterations as f64;
|
||||
started.elapsed()
|
||||
}
|
||||
|
||||
fn measure(name: &str, minimum_iterations: u64, mut f: impl FnMut()) {
|
||||
for _ in 0..minimum_iterations.min(1000) {
|
||||
f();
|
||||
}
|
||||
let mut iterations = minimum_iterations.max(1);
|
||||
while sample(iterations, &mut f) < Duration::from_millis(20) && iterations < 100_000_000 {
|
||||
iterations = (iterations * 2).min(100_000_000);
|
||||
}
|
||||
let mut measurements = [0_f64; 7];
|
||||
for value in &mut measurements {
|
||||
*value = sample(iterations, &mut f).as_nanos() as f64 / iterations as f64;
|
||||
}
|
||||
measurements.sort_by(f64::total_cmp);
|
||||
let ns = measurements[measurements.len() / 2];
|
||||
println!("BENCH {name} {ns:.3} ns/op");
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user