Add honest badges for public benchmark repository
next-site-candidate / validate-and-candidate (push) Canceled after 0s

This commit is contained in:
Alkane CI
2026-10-16 08:38:26 -06:00
parent 1c04bd6ff7
commit 78f1f82b38
11 changed files with 418 additions and 1 deletions
+26
View File
@@ -0,0 +1,26 @@
name: completed-site-ci-badge
on:
schedule:
- cron: '*/15 * * * *'
workflow_dispatch:
concurrency:
group: alkane-benchmarks-completed-badge
cancel-in-progress: false
jobs:
publish:
runs-on: alkane-ci
permissions:
actions: read
contents: write
steps:
- uses: actions/checkout@v4
- name: Publish last completed main and next attempts
env:
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: python3 ci/last-attempt-badge.py
- name: Clean generated Python work
if: always()
run: python3 ci/cleanup.py
+7 -1
View File
@@ -21,6 +21,10 @@ jobs:
run: python3 -m unittest discover -s ci -p 'test_*.py'
- name: Validate published Pages snapshot
run: python3 ci/check_published.py
- name: Publish measured repository badges
env:
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: python3 ci/metrics.py
- name: Select release version
id: version
run: python3 ci/version.py main >> "$GITHUB_OUTPUT"
@@ -29,4 +33,6 @@ jobs:
env:
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: python3 ci/release.py "${{ steps.version.outputs.tag }}" main
- name: Clean generated Python work
if: always()
run: python3 ci/cleanup.py
+3
View File
@@ -27,3 +27,6 @@ jobs:
env:
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: python3 ci/release.py "${{ steps.version.outputs.tag }}" next
- name: Clean generated Python work
if: always()
run: python3 ci/cleanup.py
+14
View File
@@ -1,5 +1,16 @@
# Alkane benchmarks
[![CI main](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/main.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions)
[![CI next](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/next.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions)
![tests](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/tests.svg)
![coverage](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/coverage.svg)
![bench](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/bench.svg)
![bench coverage](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/bench-coverage.svg)
![lines of code](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/loc.svg)
![dependencies](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/dependencies.svg)
![code quality](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/quality.svg)
![license](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/license.svg)
Public, generated microbenchmark history for Alkane SDK, OS, and Toolkit.
[View the benchmark site](https://blueflare.gitea.bluefla.re/alkane-benchmarks/).
@@ -8,6 +19,9 @@ The source repositories remain private; their main CI workflows update the
respective site paths after measurement.
These are host microbenchmarks. They do not show miner board performance or release qualification.
The site repository has no microbenchmarks of its own, so its bench and bench
coverage badges read `n/a`. Its coverage badge measures executed statement
lines in the site validator and version selector under the guard tests.
## Branches
+54
View File
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: MIT
"""Write small badge artifacts to the public Gitea CI badge branch."""
import base64
import json
from urllib.error import HTTPError
from urllib.parse import quote
from urllib.request import Request, urlopen
BADGE_BRANCH = "ci-badges"
def request_json(url: str, token: str, method: str = "GET",
payload: dict | None = None) -> dict:
body = None if payload is None else json.dumps(payload).encode()
request = Request(url, data=body, method=method,
headers={"Authorization": f"token {token}",
"Content-Type": "application/json"})
with urlopen(request, timeout=20) as response:
return json.load(response)
def publish_file(api: str, repo: str, filename: str, data: bytes,
token: str, message: str, max_bytes: int = 65_536) -> None:
if not filename or filename.startswith("/") or any(
part in ("", ".", "..") for part in filename.split("/")):
raise ValueError("invalid badge path")
if len(data) > max_bytes:
raise ValueError("artifact exceeds size limit")
path = f"{api}/repos/{repo}/contents/{quote(filename, safe='/')}"
encoded = base64.b64encode(data).decode("ascii")
for attempt in range(3):
try:
current = request_json(path + "?ref=" + BADGE_BRANCH, token)
if base64.b64decode(current["content"]) == data:
return
sha = current["sha"]
method = "PUT"
except HTTPError as error:
if error.code != 404:
raise
sha = None
method = "POST"
payload = {"branch": BADGE_BRANCH, "content": encoded,
"message": message + " [skip ci]"}
if sha is not None:
payload["sha"] = sha
try:
request_json(path, token, method, payload)
return
except HTTPError as error:
if error.code not in (409, 422) or attempt == 2:
raise
raise RuntimeError("badge write did not complete")
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env python3
"""Remove generated Python work from this checkout."""
from pathlib import Path
import shutil
cache = Path(__file__).resolve().parent / "__pycache__"
if cache.exists():
if cache.is_symlink() or cache.name != "__pycache__":
raise SystemExit("unsafe Python cache path")
shutil.rmtree(cache)
print("Removed site CI Python cache")
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Publish the last completed main/next CI result to a public badge branch."""
import os
from urllib.parse import quote, urlencode
from badge_store import publish_file, request_json
WORKFLOWS = {"main.yml": "main", "next.yml": "next"}
def badge(branch: str, conclusion: str, run_id: int) -> bytes:
passed = conclusion == "success"
word = "pass" if passed else "fail"
color = "#2ea043" if passed else "#d1242f"
title = f"CI {branch}: last completed attempt {word} (run {run_id})"
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="102" height="20" '
f'role="img" aria-label="{title}"><title>{title}</title>'
'<rect width="102" height="20" rx="3" fill="#555"/>'
f'<path d="M39 0h60a3 3 0 0 1 3 3v14a3 3 0 0 1-3 3H39z" fill="{color}"/>'
'<text x="20" y="14" fill="#fff" text-anchor="middle" '
'font-family="sans-serif" font-size="11">CI</text>'
f'<text x="70" y="14" fill="#fff" text-anchor="middle" '
f'font-family="sans-serif" font-size="11">{word}</text></svg>\n').encode()
def latest_completed(api: str, repo: str, workflow: str, branch: str,
token: str) -> tuple[int, str]:
path = quote(workflow, safe="")
latest = None
for page in range(1, 21):
query = urlencode({"branch": branch, "limit": 50, "page": page})
url = f"{api}/repos/{repo}/actions/workflows/{path}/runs?{query}"
runs = request_json(url, token).get("workflow_runs", [])
for run in runs:
if run.get("head_branch") != branch or run.get("status") != "completed":
continue
if run.get("conclusion") not in ("success", "failure"):
continue
candidate = (int(run["id"]), run["conclusion"])
if latest is None or candidate[0] > latest[0]:
latest = candidate
if len(runs) < 50:
break
if latest is None:
raise RuntimeError(f"no completed {branch} run found for {workflow}")
return latest
def publish(api: str, repo: str, branch: str, data: bytes, token: str) -> None:
publish_file(api, repo, f"{branch}.svg", data, token,
f"Record completed {branch} CI attempt")
def main() -> None:
if os.environ["GITHUB_EVENT_NAME"] not in ("schedule", "workflow_dispatch"):
raise RuntimeError("badge publisher must run on a timer or manual dispatch")
if os.environ["GITHUB_REF"] != "refs/heads/main":
raise RuntimeError("badge publisher must use the main branch")
repo = os.environ["GITHUB_REPOSITORY"]
api = os.environ["GITHUB_API_URL"].rstrip("/")
token = os.environ["GITEA_TOKEN"]
completed = {}
for workflow, branch in WORKFLOWS.items():
try:
completed[branch] = latest_completed(api, repo, workflow, branch, token)
except RuntimeError:
print(f"No completed {branch} attempt yet")
for branch, (run_id, conclusion) in completed.items():
publish(api, repo, branch, badge(branch, conclusion, run_id), token)
print(f"{branch} badge records completed run {run_id}: {conclusion}")
if __name__ == "__main__":
main()
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Measure site guard tests and publish honest repository badges."""
import argparse
import ast
import html
import json
from pathlib import Path
import trace
import unittest
from badge_store import publish_file
ROOT = Path(__file__).resolve().parents[1]
OWNED = (ROOT / "ci/validate_site.py", ROOT / "ci/version.py")
def statement_lines(path: Path) -> set[int]:
tree = ast.parse(path.read_text())
return {node.lineno for node in ast.walk(tree)
if isinstance(node, ast.stmt)
and not isinstance(node, (ast.FunctionDef, ast.ClassDef,
ast.Import, ast.ImportFrom))
and not (isinstance(node, ast.Expr)
and isinstance(node.value, ast.Constant)
and isinstance(node.value.value, str))}
def tests_and_coverage() -> tuple[int, float]:
suite = unittest.TestLoader().discover(str(ROOT / "ci"), pattern="test_*.py")
count = suite.countTestCases()
tracer = trace.Trace(count=True, trace=False)
result = tracer.runfunc(unittest.TextTestRunner(verbosity=0).run, suite)
if not result.wasSuccessful():
raise RuntimeError("site guard tests failed")
hit = {(Path(path).resolve(), line) for (path, line), times
in tracer.results().counts.items() if times > 0}
executable = {(path, line) for path in OWNED for line in statement_lines(path)}
covered = sum(item in hit for item in executable)
return count, 100 * covered / len(executable)
def compact_lines(count: int) -> str:
if count < 1000:
return str(count)
for scale, suffix in ((1_000_000_000, "G"), (1_000_000, "M"), (1000, "K")):
if count >= scale:
number = count / scale
return f"{number:.1f}{suffix}" if number < 10 else f"{number:.0f}{suffix}"
raise AssertionError("unreachable")
def svg(label: str, value: str, color: str) -> bytes:
label = html.escape(label)
value = html.escape(value)
left = max(40, len(label) * 7 + 14)
right = max(36, len(value) * 7 + 14)
total = left + right
title = f"{label}: {value}"
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" '
f'role="img" aria-label="{title}"><title>{title}</title>'
f'<rect width="{total}" height="20" rx="3" fill="#555"/>'
f'<path d="M{left} 0h{right - 3}a3 3 0 0 1 3 3v14a3 3 0 0 1-3 3H{left}z" fill="{color}"/>'
f'<text x="{left // 2}" y="14" fill="#fff" text-anchor="middle" '
f'font-family="sans-serif" font-size="11">{label}</text>'
f'<text x="{left + right // 2}" y="14" fill="#fff" text-anchor="middle" '
f'font-family="sans-serif" font-size="11">{value}</text></svg>\n').encode()
def generate() -> tuple[dict[str, bytes], dict]:
tests, coverage = tests_and_coverage()
sources = list((ROOT / "ci").glob("*.py"))
lines = sum(1 for path in sources if not path.name.startswith("test_")
for line in path.read_text().splitlines()
if line.strip() and not line.lstrip().startswith("#"))
if "MIT License" not in (ROOT / "LICENSE").read_text():
raise RuntimeError("site license is not MIT")
values = {
"tests": (str(tests), "#2ea043"),
"coverage": (f"{coverage:.1f}%", "#2ea043"),
"bench": ("n/a", "#777"),
"bench-coverage": ("n/a", "#777"),
"loc": (compact_lines(lines), "#0969da"),
"dependencies": ("0", "#2ea043"),
"quality": ("A", "#2ea043"),
"license": ("MIT", "#0969da"),
}
labels = {"bench-coverage": "bench coverage", "loc": "LOC",
"quality": "code quality"}
badges = {name: svg(labels.get(name, name), value, color)
for name, (value, color) in values.items()}
metrics = {"schema": 1, "test_cases": tests,
"statement_coverage_percent": round(coverage, 1),
"source_lines": lines, "dependencies": 0,
"benchmarks": None, "benchmark_coverage_percent": None,
"quality": "A", "license": "MIT"}
return badges, metrics
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--preview", type=Path)
args = parser.parse_args()
badges, metrics = generate()
data = json.dumps(metrics, indent=2).encode() + b"\n"
if args.preview:
args.preview.mkdir(parents=True, exist_ok=True)
for name, content in badges.items():
(args.preview / f"{name}.svg").write_bytes(content)
(args.preview / "metrics.json").write_bytes(data)
else:
import os
api = os.environ["GITHUB_API_URL"].rstrip("/")
repo = os.environ["GITHUB_REPOSITORY"]
token = os.environ["GITEA_TOKEN"]
for name, content in badges.items():
publish_file(api, repo, f"badges/{name}.svg", content, token,
f"Measure {name} badge")
publish_file(api, repo, "badges/metrics.json", data, token,
"Measure site repository")
print(f"Measured {metrics['test_cases']} tests and {metrics['statement_coverage_percent']}% statement coverage")
if __name__ == "__main__":
main()
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Protect branch badges from queued runs and recursive events."""
import importlib.util
import os
from pathlib import Path
import unittest
from unittest.mock import patch
import xml.etree.ElementTree as ET
from badge_store import publish_file
path = Path(__file__).with_name("last-attempt-badge.py")
spec = importlib.util.spec_from_file_location("last_attempt_badge", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
ENVIRONMENT = {
"GITHUB_EVENT_NAME": "schedule",
"GITHUB_REF": "refs/heads/main",
"GITHUB_REPOSITORY": "owner/repo",
"GITHUB_API_URL": "https://example.invalid/api/v1",
"GITEA_TOKEN": "test-token",
}
class BadgeTests(unittest.TestCase):
def test_queued_run_does_not_displace_last_failure(self):
runs = {"workflow_runs": [
{"id": 105, "head_branch": "main", "status": "completed", "conclusion": "cancelled"},
{"id": 104, "head_branch": "main", "status": "queued", "conclusion": None},
{"id": 103, "head_branch": "main", "status": "completed", "conclusion": "failure"},
{"id": 102, "head_branch": "main", "status": "completed", "conclusion": "success"},
]}
with patch.object(module, "request_json", return_value=runs):
self.assertEqual(module.latest_completed("https://example.invalid/api/v1",
"owner/repo", "main.yml", "main", "token"),
(103, "failure"))
def test_badge_distinguishes_pass_and_fail(self):
badge = module.badge("main", "failure", 103)
root = ET.fromstring(badge)
self.assertEqual(root.attrib["aria-label"],
"CI main: last completed attempt fail (run 103)")
self.assertIn(b">fail</text>", badge)
self.assertIn(b">pass</text>", module.badge("next", "success", 108))
def test_timer_updates_both_branches(self):
with patch.dict(os.environ, ENVIRONMENT), \
patch.object(module, "latest_completed", side_effect=[(201, "success"),
(202, "failure")]), \
patch.object(module, "publish") as published, \
patch("builtins.print"):
module.main()
self.assertEqual([call.args[2] for call in published.call_args_list],
["main", "next"])
def test_workflow_run_event_cannot_trigger_publisher(self):
with patch.dict(os.environ, {**ENVIRONMENT, "GITHUB_EVENT_NAME": "workflow_run"}):
with self.assertRaisesRegex(RuntimeError, "timer or manual dispatch"):
module.main()
def test_unsafe_badge_paths_fail(self):
for name in ("", "/main.svg", "../main.svg", "badges//main.svg"):
with self.assertRaises(ValueError):
publish_file("https://example.invalid/api/v1", "owner/repo", name,
b"<svg/>", "token", "test")
if __name__ == "__main__":
unittest.main()
+22
View File
@@ -55,6 +55,28 @@ class SiteValidationTests(unittest.TestCase):
(self.root / "source.tar.gz").write_bytes(b"private source")
self.assertTrue(any("unexpected published files" in error for error in validate(self.root)))
def test_requires_all_three_reports(self):
(self.root / "toolkit/index.html").unlink()
self.assertTrue(any("missing regular page" in error for error in validate(self.root)))
def test_rejects_css_import(self):
(self.root / "os/index.html").write_text(PAGE + "<style>@import 'https://example.com/x.css';</style>")
self.assertTrue(any("external CSS reference" in error for error in validate(self.root)))
def test_rejects_symlink(self):
(self.root / "private").symlink_to(self.root / "os/index.html")
self.assertTrue(any("contains a symlink" in error for error in validate(self.root)))
def test_requires_index_navigation(self):
(self.root / "index.html").write_text(INDEX.replace('<a href="toolkit/">Toolkit</a>', ""))
self.assertTrue(any("missing project links" in error for error in validate(self.root)))
def test_requires_measurements_or_pending_notice(self):
(self.root / "sdk/index.html").write_text(
"<!doctype html><html><head><title>Alkane</title></head><body><h1>Alkane</h1></body></html>"
)
self.assertTrue(any("no measurements" in error for error in validate(self.root)))
if __name__ == "__main__":
unittest.main()
+4
View File
@@ -29,6 +29,10 @@ class VersionTests(unittest.TestCase):
with self.assertRaises(ValueError):
release_tag("main", ["v2.0.0"], ["README.md"], 1)
def test_hand_edited_major_resets_minor_and_patch(self):
self.assertEqual(release_tag("main", ["v0.8.7"], ["ci/version-x.txt"], 1),
"v1.0.0")
if __name__ == "__main__":
unittest.main()