From 35b96c493fde8e9a8e7779399976277ad350bece Mon Sep 17 00:00:00 2001 From: Noah <2+noah@noreply.gitea.bluefla.re> Date: Tue, 3 Nov 2026 11:03:55 -0600 Subject: [PATCH] Hand the toolkit benchmarks to the toolkit's own site The toolkit repository is public and now publishes its own Pages site (benchmarks, badges, quality dashboard), so this site covers only the private SDK and OS. - validate_site: drop the toolkit page; the index may link to the toolkit's bench page. - check_live_pages: /alkane-toolkit/ and its bench page must be public; only the SDK and OS sites must stay closed. - metrics: grade coverage on the toolkit's bands instead of a fixed green A. - README: one CI badge plus the license; the script metrics no longer pose as SDK or OS metrics. Co-Authored-By: Claude Opus 5.5 --- README.md | 24 ++++++++++-------------- ci/check_live_pages.py | 10 +++++++--- ci/metrics.py | 19 ++++++++++++++++--- ci/test_validate_site.py | 15 ++++++++++----- ci/validate_site.py | 10 ++++++---- 5 files changed, 49 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 6d38ef7..67c25cf 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,19 @@ # Alkane benchmarks -[![CI main](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/main.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions) -[![CI next](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/next.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions) -![tests](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/tests.svg) -![coverage](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/coverage.svg) -![bench](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/bench.svg) -![bench coverage](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/bench-coverage.svg) -![lines of code](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/loc.svg) -![dependencies](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/dependencies.svg) -![code quality](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/quality.svg) +[![CI](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/main.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions) ![license](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/license.svg) Public [benchmark history](https://blueflare.gitea.bluefla.re/alkane-benchmarks/) -for Alkane SDK, OS, and Toolkit. Scheduled publishers copy measured host data -after CI. Source repositories remain private. These results do not measure -miner boards or qualify a release. +for the Alkane SDK and OS, whose source repositories are private. Publishers +copy measured host data after CI. These results do not measure miner boards +or qualify a release. -This site has no microbenchmarks of its own, so its bench badges read `n/a`. -Its coverage badge measures the site validator and version selector. +The toolkit is public and publishes its own benchmarks, badges and quality +dashboard at . + +This repository holds only the site validator and publishing scripts, so it +shows no test, coverage or size badges: those would describe the scripts, not +the SDK or OS. ## Branches diff --git a/ci/check_live_pages.py b/ci/check_live_pages.py index 730fcdc..8cfaac7 100644 --- a/ci/check_live_pages.py +++ b/ci/check_live_pages.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MIT -"""Check that benchmark Pages are public and source repository Pages stay closed.""" +"""Check that benchmark Pages are public and private source repositories stay closed. + +The toolkit repository is public and serves its own site, benchmarks included. +""" from urllib.error import HTTPError, URLError from urllib.request import HTTPRedirectHandler, Request, build_opener @@ -11,9 +14,10 @@ PUBLIC = ( "/alkane-benchmarks/", "/alkane-benchmarks/sdk/", "/alkane-benchmarks/os/", - "/alkane-benchmarks/toolkit/", + "/alkane-toolkit/", + "/alkane-toolkit/bench/", ) -PRIVATE = ("/alkane-sdk/", "/alkane-os/", "/alkane-toolkit/") +PRIVATE = ("/alkane-sdk/", "/alkane-os/") class NoRedirect(HTTPRedirectHandler): diff --git a/ci/metrics.py b/ci/metrics.py index dadd2af..fc3a2f1 100644 --- a/ci/metrics.py +++ b/ci/metrics.py @@ -68,6 +68,18 @@ def svg(label: str, value: str, color: str) -> bytes: f'font-family="sans-serif" font-size="11">{value}\n').encode() +# Same bands as the toolkit's quality grade: coverage sets the grade. +GRADE_BANDS = ((80.0, "A", "#2ea043"), (60.0, "B", "#7cb342"), (40.0, "C", "#d4a72c"), + (20.0, "D", "#e16f24")) + + +def grade(coverage: float) -> tuple[str, str]: + for floor, letter, color in GRADE_BANDS: + if coverage >= floor: + return letter, color + return "F", "#cf222e" + + def generate() -> tuple[dict[str, bytes], dict]: tests, coverage = tests_and_coverage() sources = list((ROOT / "ci").glob("*.py")) @@ -76,14 +88,15 @@ def generate() -> tuple[dict[str, bytes], dict]: if line.strip() and not line.lstrip().startswith("#")) if "MIT License" not in (ROOT / "LICENSE").read_text(): raise RuntimeError("site license is not MIT") + letter, color = grade(coverage) values = { "tests": (str(tests), "#2ea043"), - "coverage": (f"{coverage:.1f}%", "#2ea043"), + "coverage": (f"{coverage:.1f}%", color), "bench": ("n/a", "#777"), "bench-coverage": ("n/a", "#777"), "loc": (compact_lines(lines), "#0969da"), "dependencies": ("0", "#2ea043"), - "quality": ("A", "#2ea043"), + "quality": (letter, color), "license": ("MIT", "#0969da"), } labels = {"bench-coverage": "bench coverage", "loc": "LOC", @@ -94,7 +107,7 @@ def generate() -> tuple[dict[str, bytes], dict]: "statement_coverage_percent": round(coverage, 1), "source_lines": lines, "dependencies": 0, "benchmarks": None, "benchmark_coverage_percent": None, - "quality": "A", "license": "MIT"} + "quality": letter, "license": "MIT"} return badges, metrics diff --git a/ci/test_validate_site.py b/ci/test_validate_site.py index 5d924a1..d1057e9 100644 --- a/ci/test_validate_site.py +++ b/ci/test_validate_site.py @@ -12,7 +12,7 @@ from validate_site import validate INDEX = """Alkane benchmarks

Alkane benchmarks

SDK -OSToolkit""" +OS""" PAGE = """Alkane benchmarks

Alkane benchmarks

10 ns/op
""" @@ -23,7 +23,7 @@ class SiteValidationTests(unittest.TestCase): self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) (self.root / "index.html").write_text(INDEX) - for repo in ("sdk", "os", "toolkit"): + for repo in ("sdk", "os"): (self.root / repo).mkdir() (self.root / repo / "index.html").write_text(PAGE) @@ -60,12 +60,17 @@ class SiteValidationTests(unittest.TestCase): ) self.assertTrue(any("external or active reference" in error for error in validate(self.root))) + def test_allows_link_to_toolkit_site(self): + (self.root / "index.html").write_text(INDEX.replace( + "", 'Toolkit')) + self.assertEqual(validate(self.root), []) + def test_rejects_extra_files(self): (self.root / "source.tar.gz").write_bytes(b"private source") self.assertTrue(any("unexpected published files" in error for error in validate(self.root))) - def test_requires_all_three_reports(self): - (self.root / "toolkit/index.html").unlink() + def test_requires_both_reports(self): + (self.root / "os/index.html").unlink() self.assertTrue(any("missing regular page" in error for error in validate(self.root))) def test_rejects_css_import(self): @@ -103,7 +108,7 @@ class SiteValidationTests(unittest.TestCase): self.assertTrue(any("contains a symlink" in error for error in validate(self.root))) def test_requires_index_navigation(self): - (self.root / "index.html").write_text(INDEX.replace('Toolkit', "")) + (self.root / "index.html").write_text(INDEX.replace('OS', "")) self.assertTrue(any("missing project links" in error for error in validate(self.root))) def test_requires_measurements_or_pending_notice(self): diff --git a/ci/validate_site.py b/ci/validate_site.py index f6a6947..7d3eb75 100644 --- a/ci/validate_site.py +++ b/ci/validate_site.py @@ -7,13 +7,15 @@ from pathlib import Path import re -PAGES = ("index.html", "sdk/index.html", "os/index.html", "toolkit/index.html") +PAGES = ("index.html", "sdk/index.html", "os/index.html") +# The toolkit is public and publishes its own site; the index links to it. +TOOLKIT_SITE = "https://blueflare.gitea.bluefla.re/alkane-toolkit/bench/" ALLOWED_TAGS = {"html", "head", "meta", "title", "style", "body", "h1", "p", "a", "ul", "li", "div", "small", "table", "thead", "tbody", "tr", "th", "td"} ALLOWED_ATTRIBUTES = {"html": {"lang"}, "a": {"href"}, "div": {"class"}, "th": {"scope"}} -LOCAL_LINKS = {"sdk/", "os/", "toolkit/", "../"} +LOCAL_LINKS = {"sdk/", "os/", "../", TOOLKIT_SITE} FORBIDDEN_TEXT = ("BEGIN PRIVATE KEY", "GITEA_TOKEN", "ghp_", "glpat-") COMMIT_LINK = re.compile( r"https://gitea\.bluefla\.re/blueflare/alkane-(?:sdk|os|toolkit)/commit/[0-9a-f]{40}" @@ -86,7 +88,7 @@ def validate_page(path: Path, relative: str) -> list[str]: if re.search(r"url\s*\(|@import", text, re.IGNORECASE): errors.append(f"{relative}: external CSS reference is not allowed") if relative == "index.html": - if not {"sdk/", "os/", "toolkit/"}.issubset(parser.links): + if not {"sdk/", "os/"}.issubset(parser.links): errors.append("index.html: missing project links") elif not parser.tables and "Waiting for a completed measured CI run." not in text: errors.append(f"{relative}: no measurements or pending notice") @@ -111,7 +113,7 @@ def validate(root: Path) -> list[str]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("site", type=Path) - parser.add_argument("--page", choices=("sdk", "os", "toolkit"), + parser.add_argument("--page", choices=("sdk", "os"), help="validate one generated report before publication") args = parser.parse_args() errors = (validate_page(args.site, f"{args.page}/index.html")