Add honest badges for public benchmark repository
next-site-candidate / validate-and-candidate (push) Canceled after 0s
next-site-candidate / validate-and-candidate (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
name: completed-site-ci-badge
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '*/15 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: alkane-benchmarks-completed-badge
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: alkane-ci
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Publish last completed main and next attempts
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: python3 ci/last-attempt-badge.py
|
||||
- name: Clean generated Python work
|
||||
if: always()
|
||||
run: python3 ci/cleanup.py
|
||||
@@ -21,6 +21,10 @@ jobs:
|
||||
run: python3 -m unittest discover -s ci -p 'test_*.py'
|
||||
- name: Validate published Pages snapshot
|
||||
run: python3 ci/check_published.py
|
||||
- name: Publish measured repository badges
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: python3 ci/metrics.py
|
||||
- name: Select release version
|
||||
id: version
|
||||
run: python3 ci/version.py main >> "$GITHUB_OUTPUT"
|
||||
@@ -29,4 +33,6 @@ jobs:
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: python3 ci/release.py "${{ steps.version.outputs.tag }}" main
|
||||
|
||||
- name: Clean generated Python work
|
||||
if: always()
|
||||
run: python3 ci/cleanup.py
|
||||
|
||||
@@ -27,3 +27,6 @@ jobs:
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: python3 ci/release.py "${{ steps.version.outputs.tag }}" next
|
||||
- name: Clean generated Python work
|
||||
if: always()
|
||||
run: python3 ci/cleanup.py
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
# Alkane benchmarks
|
||||
|
||||
[](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions)
|
||||
[](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions)
|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
Public, generated microbenchmark history for Alkane SDK, OS, and Toolkit.
|
||||
|
||||
[View the benchmark site](https://blueflare.gitea.bluefla.re/alkane-benchmarks/).
|
||||
@@ -8,6 +19,9 @@ The source repositories remain private; their main CI workflows update the
|
||||
respective site paths after measurement.
|
||||
|
||||
These are host microbenchmarks. They do not show miner board performance or release qualification.
|
||||
The site repository has no microbenchmarks of its own, so its bench and bench
|
||||
coverage badges read `n/a`. Its coverage badge measures executed statement
|
||||
lines in the site validator and version selector under the guard tests.
|
||||
|
||||
## Branches
|
||||
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Write small badge artifacts to the public Gitea CI badge branch."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
BADGE_BRANCH = "ci-badges"
|
||||
|
||||
|
||||
def request_json(url: str, token: str, method: str = "GET",
|
||||
payload: dict | None = None) -> dict:
|
||||
body = None if payload is None else json.dumps(payload).encode()
|
||||
request = Request(url, data=body, method=method,
|
||||
headers={"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json"})
|
||||
with urlopen(request, timeout=20) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def publish_file(api: str, repo: str, filename: str, data: bytes,
|
||||
token: str, message: str, max_bytes: int = 65_536) -> None:
|
||||
if not filename or filename.startswith("/") or any(
|
||||
part in ("", ".", "..") for part in filename.split("/")):
|
||||
raise ValueError("invalid badge path")
|
||||
if len(data) > max_bytes:
|
||||
raise ValueError("artifact exceeds size limit")
|
||||
path = f"{api}/repos/{repo}/contents/{quote(filename, safe='/')}"
|
||||
encoded = base64.b64encode(data).decode("ascii")
|
||||
for attempt in range(3):
|
||||
try:
|
||||
current = request_json(path + "?ref=" + BADGE_BRANCH, token)
|
||||
if base64.b64decode(current["content"]) == data:
|
||||
return
|
||||
sha = current["sha"]
|
||||
method = "PUT"
|
||||
except HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
sha = None
|
||||
method = "POST"
|
||||
payload = {"branch": BADGE_BRANCH, "content": encoded,
|
||||
"message": message + " [skip ci]"}
|
||||
if sha is not None:
|
||||
payload["sha"] = sha
|
||||
try:
|
||||
request_json(path, token, method, payload)
|
||||
return
|
||||
except HTTPError as error:
|
||||
if error.code not in (409, 422) or attempt == 2:
|
||||
raise
|
||||
raise RuntimeError("badge write did not complete")
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Remove generated Python work from this checkout."""
|
||||
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
|
||||
cache = Path(__file__).resolve().parent / "__pycache__"
|
||||
if cache.exists():
|
||||
if cache.is_symlink() or cache.name != "__pycache__":
|
||||
raise SystemExit("unsafe Python cache path")
|
||||
shutil.rmtree(cache)
|
||||
print("Removed site CI Python cache")
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Publish the last completed main/next CI result to a public badge branch."""
|
||||
|
||||
import os
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
from badge_store import publish_file, request_json
|
||||
|
||||
|
||||
WORKFLOWS = {"main.yml": "main", "next.yml": "next"}
|
||||
|
||||
|
||||
def badge(branch: str, conclusion: str, run_id: int) -> bytes:
|
||||
passed = conclusion == "success"
|
||||
word = "pass" if passed else "fail"
|
||||
color = "#2ea043" if passed else "#d1242f"
|
||||
title = f"CI {branch}: last completed attempt {word} (run {run_id})"
|
||||
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="102" height="20" '
|
||||
f'role="img" aria-label="{title}"><title>{title}</title>'
|
||||
'<rect width="102" height="20" rx="3" fill="#555"/>'
|
||||
f'<path d="M39 0h60a3 3 0 0 1 3 3v14a3 3 0 0 1-3 3H39z" fill="{color}"/>'
|
||||
'<text x="20" y="14" fill="#fff" text-anchor="middle" '
|
||||
'font-family="sans-serif" font-size="11">CI</text>'
|
||||
f'<text x="70" y="14" fill="#fff" text-anchor="middle" '
|
||||
f'font-family="sans-serif" font-size="11">{word}</text></svg>\n').encode()
|
||||
|
||||
|
||||
def latest_completed(api: str, repo: str, workflow: str, branch: str,
|
||||
token: str) -> tuple[int, str]:
|
||||
path = quote(workflow, safe="")
|
||||
latest = None
|
||||
for page in range(1, 21):
|
||||
query = urlencode({"branch": branch, "limit": 50, "page": page})
|
||||
url = f"{api}/repos/{repo}/actions/workflows/{path}/runs?{query}"
|
||||
runs = request_json(url, token).get("workflow_runs", [])
|
||||
for run in runs:
|
||||
if run.get("head_branch") != branch or run.get("status") != "completed":
|
||||
continue
|
||||
if run.get("conclusion") not in ("success", "failure"):
|
||||
continue
|
||||
candidate = (int(run["id"]), run["conclusion"])
|
||||
if latest is None or candidate[0] > latest[0]:
|
||||
latest = candidate
|
||||
if len(runs) < 50:
|
||||
break
|
||||
if latest is None:
|
||||
raise RuntimeError(f"no completed {branch} run found for {workflow}")
|
||||
return latest
|
||||
|
||||
|
||||
def publish(api: str, repo: str, branch: str, data: bytes, token: str) -> None:
|
||||
publish_file(api, repo, f"{branch}.svg", data, token,
|
||||
f"Record completed {branch} CI attempt")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if os.environ["GITHUB_EVENT_NAME"] not in ("schedule", "workflow_dispatch"):
|
||||
raise RuntimeError("badge publisher must run on a timer or manual dispatch")
|
||||
if os.environ["GITHUB_REF"] != "refs/heads/main":
|
||||
raise RuntimeError("badge publisher must use the main branch")
|
||||
repo = os.environ["GITHUB_REPOSITORY"]
|
||||
api = os.environ["GITHUB_API_URL"].rstrip("/")
|
||||
token = os.environ["GITEA_TOKEN"]
|
||||
completed = {}
|
||||
for workflow, branch in WORKFLOWS.items():
|
||||
try:
|
||||
completed[branch] = latest_completed(api, repo, workflow, branch, token)
|
||||
except RuntimeError:
|
||||
print(f"No completed {branch} attempt yet")
|
||||
for branch, (run_id, conclusion) in completed.items():
|
||||
publish(api, repo, branch, badge(branch, conclusion, run_id), token)
|
||||
print(f"{branch} badge records completed run {run_id}: {conclusion}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure site guard tests and publish honest repository badges."""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
import trace
|
||||
import unittest
|
||||
|
||||
from badge_store import publish_file
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OWNED = (ROOT / "ci/validate_site.py", ROOT / "ci/version.py")
|
||||
|
||||
|
||||
def statement_lines(path: Path) -> set[int]:
|
||||
tree = ast.parse(path.read_text())
|
||||
return {node.lineno for node in ast.walk(tree)
|
||||
if isinstance(node, ast.stmt)
|
||||
and not isinstance(node, (ast.FunctionDef, ast.ClassDef,
|
||||
ast.Import, ast.ImportFrom))
|
||||
and not (isinstance(node, ast.Expr)
|
||||
and isinstance(node.value, ast.Constant)
|
||||
and isinstance(node.value.value, str))}
|
||||
|
||||
|
||||
def tests_and_coverage() -> tuple[int, float]:
|
||||
suite = unittest.TestLoader().discover(str(ROOT / "ci"), pattern="test_*.py")
|
||||
count = suite.countTestCases()
|
||||
tracer = trace.Trace(count=True, trace=False)
|
||||
result = tracer.runfunc(unittest.TextTestRunner(verbosity=0).run, suite)
|
||||
if not result.wasSuccessful():
|
||||
raise RuntimeError("site guard tests failed")
|
||||
hit = {(Path(path).resolve(), line) for (path, line), times
|
||||
in tracer.results().counts.items() if times > 0}
|
||||
executable = {(path, line) for path in OWNED for line in statement_lines(path)}
|
||||
covered = sum(item in hit for item in executable)
|
||||
return count, 100 * covered / len(executable)
|
||||
|
||||
|
||||
def compact_lines(count: int) -> str:
|
||||
if count < 1000:
|
||||
return str(count)
|
||||
for scale, suffix in ((1_000_000_000, "G"), (1_000_000, "M"), (1000, "K")):
|
||||
if count >= scale:
|
||||
number = count / scale
|
||||
return f"{number:.1f}{suffix}" if number < 10 else f"{number:.0f}{suffix}"
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def svg(label: str, value: str, color: str) -> bytes:
|
||||
label = html.escape(label)
|
||||
value = html.escape(value)
|
||||
left = max(40, len(label) * 7 + 14)
|
||||
right = max(36, len(value) * 7 + 14)
|
||||
total = left + right
|
||||
title = f"{label}: {value}"
|
||||
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" '
|
||||
f'role="img" aria-label="{title}"><title>{title}</title>'
|
||||
f'<rect width="{total}" height="20" rx="3" fill="#555"/>'
|
||||
f'<path d="M{left} 0h{right - 3}a3 3 0 0 1 3 3v14a3 3 0 0 1-3 3H{left}z" fill="{color}"/>'
|
||||
f'<text x="{left // 2}" y="14" fill="#fff" text-anchor="middle" '
|
||||
f'font-family="sans-serif" font-size="11">{label}</text>'
|
||||
f'<text x="{left + right // 2}" y="14" fill="#fff" text-anchor="middle" '
|
||||
f'font-family="sans-serif" font-size="11">{value}</text></svg>\n').encode()
|
||||
|
||||
|
||||
def generate() -> tuple[dict[str, bytes], dict]:
|
||||
tests, coverage = tests_and_coverage()
|
||||
sources = list((ROOT / "ci").glob("*.py"))
|
||||
lines = sum(1 for path in sources if not path.name.startswith("test_")
|
||||
for line in path.read_text().splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#"))
|
||||
if "MIT License" not in (ROOT / "LICENSE").read_text():
|
||||
raise RuntimeError("site license is not MIT")
|
||||
values = {
|
||||
"tests": (str(tests), "#2ea043"),
|
||||
"coverage": (f"{coverage:.1f}%", "#2ea043"),
|
||||
"bench": ("n/a", "#777"),
|
||||
"bench-coverage": ("n/a", "#777"),
|
||||
"loc": (compact_lines(lines), "#0969da"),
|
||||
"dependencies": ("0", "#2ea043"),
|
||||
"quality": ("A", "#2ea043"),
|
||||
"license": ("MIT", "#0969da"),
|
||||
}
|
||||
labels = {"bench-coverage": "bench coverage", "loc": "LOC",
|
||||
"quality": "code quality"}
|
||||
badges = {name: svg(labels.get(name, name), value, color)
|
||||
for name, (value, color) in values.items()}
|
||||
metrics = {"schema": 1, "test_cases": tests,
|
||||
"statement_coverage_percent": round(coverage, 1),
|
||||
"source_lines": lines, "dependencies": 0,
|
||||
"benchmarks": None, "benchmark_coverage_percent": None,
|
||||
"quality": "A", "license": "MIT"}
|
||||
return badges, metrics
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--preview", type=Path)
|
||||
args = parser.parse_args()
|
||||
badges, metrics = generate()
|
||||
data = json.dumps(metrics, indent=2).encode() + b"\n"
|
||||
if args.preview:
|
||||
args.preview.mkdir(parents=True, exist_ok=True)
|
||||
for name, content in badges.items():
|
||||
(args.preview / f"{name}.svg").write_bytes(content)
|
||||
(args.preview / "metrics.json").write_bytes(data)
|
||||
else:
|
||||
import os
|
||||
api = os.environ["GITHUB_API_URL"].rstrip("/")
|
||||
repo = os.environ["GITHUB_REPOSITORY"]
|
||||
token = os.environ["GITEA_TOKEN"]
|
||||
for name, content in badges.items():
|
||||
publish_file(api, repo, f"badges/{name}.svg", content, token,
|
||||
f"Measure {name} badge")
|
||||
publish_file(api, repo, "badges/metrics.json", data, token,
|
||||
"Measure site repository")
|
||||
print(f"Measured {metrics['test_cases']} tests and {metrics['statement_coverage_percent']}% statement coverage")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Protect branch badges from queued runs and recursive events."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from badge_store import publish_file
|
||||
|
||||
|
||||
path = Path(__file__).with_name("last-attempt-badge.py")
|
||||
spec = importlib.util.spec_from_file_location("last_attempt_badge", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
ENVIRONMENT = {
|
||||
"GITHUB_EVENT_NAME": "schedule",
|
||||
"GITHUB_REF": "refs/heads/main",
|
||||
"GITHUB_REPOSITORY": "owner/repo",
|
||||
"GITHUB_API_URL": "https://example.invalid/api/v1",
|
||||
"GITEA_TOKEN": "test-token",
|
||||
}
|
||||
|
||||
|
||||
class BadgeTests(unittest.TestCase):
|
||||
def test_queued_run_does_not_displace_last_failure(self):
|
||||
runs = {"workflow_runs": [
|
||||
{"id": 105, "head_branch": "main", "status": "completed", "conclusion": "cancelled"},
|
||||
{"id": 104, "head_branch": "main", "status": "queued", "conclusion": None},
|
||||
{"id": 103, "head_branch": "main", "status": "completed", "conclusion": "failure"},
|
||||
{"id": 102, "head_branch": "main", "status": "completed", "conclusion": "success"},
|
||||
]}
|
||||
with patch.object(module, "request_json", return_value=runs):
|
||||
self.assertEqual(module.latest_completed("https://example.invalid/api/v1",
|
||||
"owner/repo", "main.yml", "main", "token"),
|
||||
(103, "failure"))
|
||||
|
||||
def test_badge_distinguishes_pass_and_fail(self):
|
||||
badge = module.badge("main", "failure", 103)
|
||||
root = ET.fromstring(badge)
|
||||
self.assertEqual(root.attrib["aria-label"],
|
||||
"CI main: last completed attempt fail (run 103)")
|
||||
self.assertIn(b">fail</text>", badge)
|
||||
self.assertIn(b">pass</text>", module.badge("next", "success", 108))
|
||||
|
||||
def test_timer_updates_both_branches(self):
|
||||
with patch.dict(os.environ, ENVIRONMENT), \
|
||||
patch.object(module, "latest_completed", side_effect=[(201, "success"),
|
||||
(202, "failure")]), \
|
||||
patch.object(module, "publish") as published, \
|
||||
patch("builtins.print"):
|
||||
module.main()
|
||||
self.assertEqual([call.args[2] for call in published.call_args_list],
|
||||
["main", "next"])
|
||||
|
||||
def test_workflow_run_event_cannot_trigger_publisher(self):
|
||||
with patch.dict(os.environ, {**ENVIRONMENT, "GITHUB_EVENT_NAME": "workflow_run"}):
|
||||
with self.assertRaisesRegex(RuntimeError, "timer or manual dispatch"):
|
||||
module.main()
|
||||
|
||||
def test_unsafe_badge_paths_fail(self):
|
||||
for name in ("", "/main.svg", "../main.svg", "badges//main.svg"):
|
||||
with self.assertRaises(ValueError):
|
||||
publish_file("https://example.invalid/api/v1", "owner/repo", name,
|
||||
b"<svg/>", "token", "test")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -55,6 +55,28 @@ class SiteValidationTests(unittest.TestCase):
|
||||
(self.root / "source.tar.gz").write_bytes(b"private source")
|
||||
self.assertTrue(any("unexpected published files" in error for error in validate(self.root)))
|
||||
|
||||
def test_requires_all_three_reports(self):
|
||||
(self.root / "toolkit/index.html").unlink()
|
||||
self.assertTrue(any("missing regular page" in error for error in validate(self.root)))
|
||||
|
||||
def test_rejects_css_import(self):
|
||||
(self.root / "os/index.html").write_text(PAGE + "<style>@import 'https://example.com/x.css';</style>")
|
||||
self.assertTrue(any("external CSS reference" in error for error in validate(self.root)))
|
||||
|
||||
def test_rejects_symlink(self):
|
||||
(self.root / "private").symlink_to(self.root / "os/index.html")
|
||||
self.assertTrue(any("contains a symlink" in error for error in validate(self.root)))
|
||||
|
||||
def test_requires_index_navigation(self):
|
||||
(self.root / "index.html").write_text(INDEX.replace('<a href="toolkit/">Toolkit</a>', ""))
|
||||
self.assertTrue(any("missing project links" in error for error in validate(self.root)))
|
||||
|
||||
def test_requires_measurements_or_pending_notice(self):
|
||||
(self.root / "sdk/index.html").write_text(
|
||||
"<!doctype html><html><head><title>Alkane</title></head><body><h1>Alkane</h1></body></html>"
|
||||
)
|
||||
self.assertTrue(any("no measurements" in error for error in validate(self.root)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -29,6 +29,10 @@ class VersionTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
release_tag("main", ["v2.0.0"], ["README.md"], 1)
|
||||
|
||||
def test_hand_edited_major_resets_minor_and_patch(self):
|
||||
self.assertEqual(release_tag("main", ["v0.8.7"], ["ci/version-x.txt"], 1),
|
||||
"v1.0.0")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user