Compare commits

...
4 Commits
7 changed files with 279 additions and 20 deletions
+25 -4
View File
@@ -48,12 +48,33 @@ jobs:
if: steps.surface.outputs.surface != 'documentation'
run: python3 ci/test-pr-surface.py
- name: Check workflow-only change
if: steps.surface.outputs.surface == 'workflow'
run: |
python3 - <<'PY'
from pathlib import Path
import yaml
for path in Path('.gitea/workflows').glob('*.yml'):
yaml.load(path.read_text(), Loader=yaml.BaseLoader)
PY
python3 ci/check-runner-image.py
for script in ci/*.sh; do bash -n "$script"; done
base="$(git merge-base HEAD "origin/$GITHUB_BASE_REF")"
git diff --check "$base" HEAD
if git grep -nIE '^[[:space:]]*-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----[[:space:]]*$' --; then
echo "private key material found in tracked source" >&2
exit 1
else
status=$?
[ "$status" -eq 1 ]
fi
- name: Install build dependencies
if: steps.surface.outputs.surface != 'documentation'
if: steps.surface.outputs.surface == 'headless' || steps.surface.outputs.surface == 'full'
run: bash ci/ensure-build-deps.sh base
- name: Configure bounded Rust compiler cache
if: steps.surface.outputs.surface != 'documentation'
if: steps.surface.outputs.surface == 'headless' || steps.surface.outputs.surface == 'full'
run: bash ci/configure-rust-cache.sh
- name: Compute wxWidgets cache key
@@ -138,7 +159,7 @@ jobs:
# runner must not fail the gate on it. Accepted advisories live in
# .cargo/audit.toml.
- name: Dependency audit (best-effort)
if: steps.surface.outputs.surface != 'documentation'
if: steps.surface.outputs.surface == 'headless' || steps.surface.outputs.surface == 'full'
continue-on-error: true
run: |
if command -v cargo-audit >/dev/null 2>&1; then
@@ -147,7 +168,7 @@ jobs:
echo "cargo-audit not installed on this runner; skipping"
fi
- name: Rust compiler cache statistics
if: always() && steps.surface.outputs.surface != 'documentation'
if: always() && (steps.surface.outputs.surface == 'headless' || steps.surface.outputs.surface == 'full')
run: |
if [ -n "${SCCACHE_DIR:-}" ]; then
sccache --show-stats
+70 -9
View File
@@ -15,6 +15,7 @@ import subprocess
import importlib.util
from urllib.error import HTTPError
from urllib.parse import quote
import yaml
from badge_store import BADGE_BRANCH, publish_file, request_json
from bench_host import identity as benchmark_host
@@ -26,6 +27,12 @@ bench_spec.loader.exec_module(bench_badge)
INDEX = "history/index.json"
MAX_INDEX_ENTRIES = 50
BENCH_INPUT_PATHS = (
"Cargo.toml", "Cargo.lock", ".cargo", "crates", "thirdparty", "vendor",
"rust-toolchain.toml", "ci/bench-badge.py", "ci/bench_host.py",
"ci/ensure-build-deps.sh", "ci/ensure-wx-prefix.sh",
"ci/configure-rust-cache.sh", "ci/wx-cache-key.sh",
)
def read_artifact(api: str, repo: str, token: str, filename: str):
@@ -51,6 +58,8 @@ def valid_benches(metrics: dict) -> dict[str, float]:
def git_distance(ancestor: str, sha: str) -> int | None:
if not re.fullmatch(r"[0-9a-f]{40}", ancestor) or not re.fullmatch(r"[0-9a-f]{40}", sha):
return None
result = subprocess.run(("git", "rev-list", "--count", f"{ancestor}..{sha}"),
capture_output=True, text=True, check=False)
if result.returncode != 0:
@@ -60,7 +69,7 @@ def git_distance(ancestor: str, sha: str) -> int | None:
return int(result.stdout.strip()) if ancestor_check.returncode == 0 else None
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
def previous_entry(index: dict, sha: str, distance=None, host=None) -> dict | None:
distance = distance or git_distance
choices = []
for entry in index.get("entries", []):
@@ -73,8 +82,41 @@ def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dic
if gap is not None and gap > 0:
choices.append((gap, entry))
if not choices:
return {}
return valid_benches(min(choices, key=lambda item: item[0])[1]["metrics"])
return None
return min(choices, key=lambda item: item[0])[1]
def previous_measurement(index: dict, sha: str, distance=None, host=None) -> dict:
entry = previous_entry(index, sha, distance, host)
return valid_benches(entry["metrics"]) if entry else {}
def benchmark_build_settings(sha: str, root: Path | str = ".") -> dict:
source = subprocess.check_output(
("git", "show", f"{sha}:.gitea/workflows/release.yml"),
cwd=root, text=True)
job = yaml.load(source, Loader=yaml.BaseLoader)["jobs"]["release"]
commands = {step.get("name", ""): step.get("run") for step in job["steps"]}
selected = ("Install build dependencies", "Configure bounded Rust compiler cache",
"Build wxWidgets (once, then cached)", "Build release", "Microbenchmarks")
return {"image": job["container"]["image"], "env": job.get("env", {}),
"commands": {name: commands.get(name) for name in selected}}
def same_benchmark_inputs(ancestor: str, sha: str, root: Path | str = ".") -> bool:
"""Identify changes that only reroute CI without changing benchmark inputs."""
if not re.fullmatch(r"[0-9a-f]{40}", ancestor) or not re.fullmatch(r"[0-9a-f]{40}", sha):
raise ValueError("invalid benchmark comparison commit")
diff = subprocess.run(("git", "diff", "--quiet", ancestor, sha, "--",
*BENCH_INPUT_PATHS), cwd=root, check=False)
if diff.returncode not in (0, 1):
raise ValueError("cannot inspect benchmark input changes")
if diff.returncode == 1:
return False
try:
return benchmark_build_settings(ancestor, root) == benchmark_build_settings(sha, root)
except (KeyError, TypeError, subprocess.CalledProcessError, yaml.YAMLError):
return False
def updated_index(index: dict, sha: str, metrics: dict, timestamp: str) -> dict:
@@ -101,6 +143,12 @@ def render_html(index: dict) -> str:
host = entry["metrics"].get("bench_host", {})
host_label = host.get("cpu_model", "unknown") if isinstance(host, dict) else "unknown"
cells.append(f'<td>{html.escape(str(host_label))}</td>')
context = entry["metrics"].get("bench_context", {})
if isinstance(context, dict) and context.get("schema") == 1:
context_label = "same inputs" if context.get("same_inputs") else "changed inputs"
else:
context_label = "unclassified"
cells.append(f'<td>{context_label}</td>')
cells += [f'<td>{values[name]:.3f}</td>' if name in values else '<td>new</td>'
for name in names]
rows.append("<tr>" + "".join(cells) + "</tr>")
@@ -115,12 +163,14 @@ def render_html(index: dict) -> str:
"th{background:#edf2f6;position:sticky;top:0}a{color:#145a9a}"
".scroll{overflow-x:auto}small{color:#55616d}</style></head><body>"
"<h1>Alkane Toolkit microbenchmarks</h1>"
"<p>Measured on main CI. Lower ns/op is faster. Results depend on runner load and are "
"not board performance. A new benchmark has no earlier comparison.</p>"
"<p>Measured on main CI. Lower ns/op is faster. Runner load affects results. "
"Same inputs marks a repeat of the benchmark source and build settings. "
"A new benchmark has no earlier comparison.</p>"
f"<p><small>Most recent {len(entries)} measured commits. Full per-commit JSON is retained "
"in the private ci-badges branch.</small></p>"
f"<div class=\"scroll\"><table><thead><tr><th scope=\"col\">Commit</th>"
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>{headings}</tr></thead><tbody>{''.join(rows)}"
f"<th scope=\"col\">UTC</th><th scope=\"col\">CPU</th>"
f"<th scope=\"col\">Inputs</th>{headings}</tr></thead><tbody>{''.join(rows)}"
"</tbody></table></div></body></html>\n")
@@ -136,10 +186,16 @@ def main() -> None:
if index.get("schema") != 1 or not isinstance(index.get("entries"), list):
raise ValueError("unsupported benchmark history index")
if args.action == "prepare":
baseline = previous_measurement(index, sha, host=benchmark_host())
entry = previous_entry(index, sha, host=benchmark_host())
baseline = valid_benches(entry["metrics"]) if entry else {}
same_inputs = same_benchmark_inputs(entry["sha"], sha) if entry else False
Path("badges").mkdir(exist_ok=True)
Path("badges/bench-baseline.json").write_text(json.dumps(baseline) + "\n")
print(f"Loaded {len(baseline)} prior benchmark measurements")
Path("badges/bench-context.json").write_text(
json.dumps({"schema": 1, "same_inputs": same_inputs,
"baseline_sha": entry["sha"] if entry else None}) + "\n")
print(f"Loaded {len(baseline)} prior benchmark measurements; "
f"same inputs: {str(same_inputs).lower()}")
return
if args.action == "render":
if not index["entries"]:
@@ -148,8 +204,13 @@ def main() -> None:
Path("bench-site/index.html").write_text(render_html(index))
print(f"Rendered {len(index['entries'])} measured benchmark commits")
return
context = json.loads(Path("badges/bench-context.json").read_text())
if (not isinstance(context, dict) or context.get("schema") != 1 or
not isinstance(context.get("same_inputs"), bool)):
raise ValueError("invalid benchmark comparison context")
metrics = {"bench_ns_per_op": bench_badge.require_suite(bench_badge.parse_bench(
Path("bench-current.txt").read_text())), "bench_host": benchmark_host()}
Path("bench-current.txt").read_text())), "bench_host": benchmark_host(),
"bench_context": context}
timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
index = updated_index(index, sha, metrics, timestamp)
record = json.dumps(index["entries"][-1], indent=2).encode() + b"\n"
+11
View File
@@ -11,6 +11,7 @@ import sys
ROOT = Path(__file__).resolve().parents[1]
DIRECTORIES = ("target", "dist", "coverage", "bench-site")
FILES = ("bench-current.txt",)
BADGE_FILES = ("bench-baseline.json", "bench-context.json")
def cleanup() -> None:
@@ -32,6 +33,16 @@ def cleanup() -> None:
if path.is_file():
path.unlink()
print(f"Removed generated file: {name}")
badge_dir = ROOT / "badges"
if badge_dir.is_symlink():
raise ValueError("unsafe generated badge directory")
for name in BADGE_FILES:
path = badge_dir / name
if path.is_symlink() or path.parent.resolve() != badge_dir.resolve():
raise ValueError(f"unsafe generated badge file: {path}")
if path.is_file():
path.unlink()
print(f"Removed generated badge file: {name}")
if __name__ == "__main__":
+15 -2
View File
@@ -11,8 +11,10 @@ Metrics not passed on the command line are computed here (LOC from the source
tree, dependency count from Cargo.lock, license from Cargo.toml).
"""
import argparse
import json
import math
import os
from pathlib import Path
import re
import sys
@@ -196,6 +198,14 @@ def main() -> int:
write("bench-coverage", "bench coverage",
f"{args.bench_coverage:.0f}%" if args.bench_coverage is not None else "pending",
cov_color(args.bench_coverage) if args.bench_coverage is not None else "lightgrey")
context_path = Path(args.root) / args.out / "bench-context.json"
same_inputs = False
if context_path.is_file():
context = json.loads(context_path.read_text())
if (not isinstance(context, dict) or context.get("schema") != 1 or
not isinstance(context.get("same_inputs"), bool)):
raise ValueError("invalid benchmark comparison context")
same_inputs = context["same_inputs"]
# Bench badge: prefer the mean per-benchmark improvement vs the previous
# release. Use the static label when no comparison is available.
delta = None
@@ -206,8 +216,11 @@ def main() -> int:
delta = None
if delta is not None:
# Positive = faster than the previous release.
msg = f"{'+' if delta > 0 else ''}{delta:.1f}% vs prev"
write("bench", "bench", msg, bench_color(delta))
msg = f"{'+' if delta > 0 else ''}{delta:.1f}%"
if same_inputs:
write("bench", "bench", msg + " same inputs", "lightgrey")
else:
write("bench", "bench", msg + " vs prev", bench_color(delta))
elif args.bench_delta == "NEW":
write("bench", "bench", "baseline set", "blue")
elif args.bench is not None:
+14 -2
View File
@@ -22,6 +22,15 @@ def documentation_only(paths: list[str]) -> bool:
)
def workflow_only(paths: list[str]) -> bool:
"""Limit the cheap PR check to workflow YAML, never policy code."""
return bool(paths) and all(
path.startswith(".gitea/workflows/") and
len(Path(path).parts) == 3 and Path(path).suffix == ".yml"
for path in paths
)
def headless_packages(paths: list[str]) -> list[str]:
"""Return tested reverse dependents for code outside the GUI graph."""
changed: set[str] = set()
@@ -95,12 +104,15 @@ def main() -> None:
try:
paths = changed_paths()
docs = documentation_only(paths)
packages = headless_packages(paths) if not docs else []
workflow = workflow_only(paths) if not docs else False
packages = headless_packages(paths) if not docs and not workflow else []
except (OSError, subprocess.CalledProcessError, ValueError):
paths = []
docs = False
workflow = False
packages = []
surface = "documentation" if docs else "headless" if packages else "full"
surface = ("documentation" if docs else "workflow" if workflow else
"headless" if packages else "full")
print(f"surface={surface}")
print(f"packages={' '.join(packages)}")
print(f"Changed paths: {len(paths)}", file=sys.stderr)
+62 -2
View File
@@ -3,11 +3,15 @@
"""Exercise benchmark history selection and safe site rendering."""
import sys
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parent))
from bench_history import bench_badge, previous_measurement, render_html, updated_index
from bench_history import (bench_badge, git_distance, previous_measurement, render_html,
same_benchmark_inputs, updated_index)
class HistoryTests(unittest.TestCase):
@@ -41,6 +45,7 @@ class HistoryTests(unittest.TestCase):
self.assertEqual(updated["entries"][-1]["metrics"]["bench_ns_per_op"]["hash"], 7.5)
def test_invalid_measurement_and_sha_rejected(self):
self.assertIsNone(git_distance("--bad", "a" * 40))
with self.assertRaises(ValueError):
updated_index({}, "bad", {"bench_ns_per_op": {"hash": 1}}, "now")
with self.assertRaises(ValueError):
@@ -56,14 +61,69 @@ class HistoryTests(unittest.TestCase):
{"hash": 10.0})
self.assertEqual(previous_measurement(index, "b" * 40, distance, other), {})
def test_routing_change_is_marked_same_inputs_but_code_or_build_change_is_not(self):
with tempfile.TemporaryDirectory(prefix="alkane-bench-inputs-") as directory:
root = Path(directory)
workflow = root / ".gitea/workflows/release.yml"
workflow.parent.mkdir(parents=True)
code = root / "crates/alkane-gui/src/main.rs"
code.parent.mkdir(parents=True)
code.write_text("fn main() {}\n")
workflow.write_text(
"jobs:\n release:\n runs-on: alkane-verify\n"
" container:\n image: pinned-image\n"
" env:\n RUSTFLAGS: stable\n steps:\n"
" - name: Build release\n run: cargo build --release\n"
" - name: Microbenchmarks\n run: cargo test --release bench_\n")
def git(*args):
return subprocess.check_output(("git", "-C", str(root), *args),
text=True, stderr=subprocess.DEVNULL).strip()
git("init", "-q")
git("config", "user.name", "CI Test")
git("config", "user.email", "ci@example.invalid")
git("add", ".")
git("commit", "-qm", "base")
base = git("rev-parse", "HEAD")
workflow.write_text(workflow.read_text().replace("alkane-verify", "alkane-ci"))
git("commit", "-qam", "route")
route = git("rev-parse", "HEAD")
self.assertTrue(same_benchmark_inputs(base, route, root))
code.write_text("fn main() { println!(\"changed\"); }\n")
git("commit", "-qam", "code")
changed_code = git("rev-parse", "HEAD")
self.assertFalse(same_benchmark_inputs(route, changed_code, root))
workflow.write_text(workflow.read_text().replace("RUSTFLAGS: stable", "RUSTFLAGS: tuned"))
git("commit", "-qam", "flags")
flags = git("rev-parse", "HEAD")
self.assertFalse(same_benchmark_inputs(changed_code, flags, root))
def test_badge_keeps_raw_delta_and_marks_identical_inputs(self):
with tempfile.TemporaryDirectory(prefix="alkane-bench-badge-context-") as directory:
root = Path(directory)
badges = root / "badges"
badges.mkdir()
(badges / "bench-context.json").write_text(
json.dumps({"schema": 1, "same_inputs": True, "baseline_sha": "a" * 40}))
command = (sys.executable, str(Path(__file__).with_name("gen-badges.py")),
"--root", str(root), "--bench-delta=-36.4")
subprocess.run(command, check=True, capture_output=True)
self.assertIn("-36.4% same inputs", (badges / "bench.svg").read_text())
(badges / "bench-context.json").unlink()
subprocess.run(command, check=True, capture_output=True)
self.assertIn("-36.4% vs prev", (badges / "bench.svg").read_text())
def test_site_escapes_history_content(self):
entry = {"sha": "a" * 40, "timestamp": "<script>alert(1)</script>",
"metrics": {"bench_ns_per_op": {"hash": 10.0},
"bench_host": {"cpu_model": "<script>cpu</script>"}}}
"bench_host": {"cpu_model": "<script>cpu</script>"},
"bench_context": {"schema": 1, "same_inputs": True}}}
page = render_html({"entries": [entry]})
self.assertIn("&lt;script&gt;", page)
self.assertNotIn("<script>", page)
self.assertIn("10.000", page)
self.assertIn("same inputs", page)
if __name__ == "__main__":
+82 -1
View File
@@ -4,13 +4,16 @@
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import importlib.util
import yaml
SCRIPT = Path(__file__).with_name("pr-surface.py")
ROOT = SCRIPT.parents[1]
spec = importlib.util.spec_from_file_location("pr_surface", SCRIPT)
assert spec and spec.loader
pr_surface = importlib.util.module_from_spec(spec)
@@ -24,8 +27,86 @@ def git(directory: Path, *args: str) -> str:
).strip()
def check_workflow_contract() -> None:
"""Keep the short workflow check from silently dropping release gates."""
required = {
"release.yml": ("main", "release", (
"Check runner image pins", "Clippy (our crate, warnings are errors)",
"Build release", "Unit tests", "Microbenchmarks",
"Headless UI self-test", "Coverage", "Microbenchmark coverage",
"Generate measured badges", "Publish measured badges",
"Record private benchmark history", "Create Gitea release + upload artifact",
"Clean generated Toolkit build",
)),
"next.yml": ("next", "candidate", (
"Check runner image pins", "Build release candidate",
"Create private prerelease", "Clean generated Toolkit build",
)),
"verify.yml": ("main", "verify", (
"Check runner image pins", "Detect changed surface",
"Check CI change surface", "Check workflow-only change",
"Clippy (our crate, warnings are errors)", "Build", "Unit tests",
"Microbenchmarks", "Headless UI self-test", "Clean generated Toolkit build",
)),
}
for filename, (branch, job_name, gates) in required.items():
path = ROOT / ".gitea/workflows" / filename
workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader)
event = "pull_request" if filename == "verify.yml" else "push"
assert workflow["on"][event]["branches"] == [branch], path
job = workflow["jobs"][job_name]
assert job["runs-on"] in {"alkane-ci", "alkane-verify"}, path
steps = job["steps"]
names = [step.get("name") for step in steps]
positions = [names.index(gate) for gate in gates]
assert positions == sorted(positions), path
assert steps[positions[-1]].get("if") == "always()", path
for gate, position in zip(gates[:-1], positions[:-1]):
condition = steps[position].get("if")
if gate in {"Create Gitea release + upload artifact",
"Create private prerelease"}:
assert condition is not None and "release == 'true'" in condition, path
elif filename == "verify.yml" and gate in {
"Build", "Unit tests", "Microbenchmarks",
"Clippy (our crate, warnings are errors)",
"Headless UI self-test"}:
assert condition == "steps.surface.outputs.surface == 'full'", path
elif filename == "verify.yml" and gate == "Check CI change surface":
assert condition == "steps.surface.outputs.surface != 'documentation'", path
elif filename == "verify.yml" and gate == "Check workflow-only change":
assert condition == "steps.surface.outputs.surface == 'workflow'", path
else:
assert condition is None, path
def main() -> None:
check_workflow_contract()
with tempfile.TemporaryDirectory(prefix="alkane-toolkit-workflow-policy-") as directory:
global ROOT
saved_root = ROOT
ROOT = Path(directory)
(ROOT / ".gitea/workflows").mkdir(parents=True)
for filename in ("release.yml", "next.yml", "verify.yml"):
shutil.copyfile(saved_root / ".gitea/workflows" / filename,
ROOT / ".gitea/workflows" / filename)
release = ROOT / ".gitea/workflows/release.yml"
original = release.read_text()
release.write_text(original.replace("- name: Coverage", "- name: Skipped coverage", 1))
try:
check_workflow_contract()
except ValueError:
pass
else:
raise AssertionError("removing the coverage gate passed workflow policy")
ROOT = saved_root
assert pr_surface.documentation_only(["README.md", "docs/setup.md"])
assert pr_surface.workflow_only([".gitea/workflows/release.yml"])
for paths in ([], ["ci/pr-surface.py"], ["ci/coverage.sh"],
["ci/check-runner-image.py"],
[".gitea/workflows/release.yml", "ci/check-runner-image.py"],
[".gitea/workflows/release.yml", "crates/alkane-gui/src/main.rs"],
[".gitea/workflows/../release.yml"]):
assert not pr_surface.workflow_only(paths), paths
for paths in ([], ["docs/generated.json"], ["src/main.rs"],
["README.md", "ci/pr-surface.py"]):
assert not pr_surface.documentation_only(paths), paths
@@ -105,7 +186,7 @@ def main() -> None:
pr_surface.ROOT = saved_root
os.environ.clear()
os.environ.update(saved_env)
print("Toolkit PR change surface: documentation, headless, and fallback PASS")
print("Toolkit PR change surface: documentation, workflow, headless, and fallback PASS")
if __name__ == "__main__":