diff --git a/README.md b/README.md index 6d38ef7..67c25cf 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,19 @@ # Alkane benchmarks -[![CI main](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/main.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions) -[![CI next](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/next.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions) -![tests](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/tests.svg) -![coverage](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/coverage.svg) -![bench](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/bench.svg) -![bench coverage](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/bench-coverage.svg) -![lines of code](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/loc.svg) -![dependencies](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/dependencies.svg) -![code quality](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/quality.svg) +[![CI](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/main.svg)](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions) ![license](https://gitea.bluefla.re/blueflare/alkane-benchmarks/raw/branch/ci-badges/badges/license.svg) Public [benchmark history](https://blueflare.gitea.bluefla.re/alkane-benchmarks/) -for Alkane SDK, OS, and Toolkit. Scheduled publishers copy measured host data -after CI. Source repositories remain private. These results do not measure -miner boards or qualify a release. +for the Alkane SDK and OS, whose source repositories are private. Publishers +copy measured host data after CI. These results do not measure miner boards +or qualify a release. -This site has no microbenchmarks of its own, so its bench badges read `n/a`. -Its coverage badge measures the site validator and version selector. +The toolkit is public and publishes its own benchmarks, badges and quality +dashboard at . + +This repository holds only the site validator and publishing scripts, so it +shows no test, coverage or size badges: those would describe the scripts, not +the SDK or OS. ## Branches diff --git a/ci/check_live_pages.py b/ci/check_live_pages.py index 730fcdc..8cfaac7 100644 --- a/ci/check_live_pages.py +++ b/ci/check_live_pages.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MIT -"""Check that benchmark Pages are public and source repository Pages stay closed.""" +"""Check that benchmark Pages are public and private source repositories stay closed. + +The toolkit repository is public and serves its own site, benchmarks included. +""" from urllib.error import HTTPError, URLError from urllib.request import HTTPRedirectHandler, Request, build_opener @@ -11,9 +14,10 @@ PUBLIC = ( "/alkane-benchmarks/", "/alkane-benchmarks/sdk/", "/alkane-benchmarks/os/", - "/alkane-benchmarks/toolkit/", + "/alkane-toolkit/", + "/alkane-toolkit/bench/", ) -PRIVATE = ("/alkane-sdk/", "/alkane-os/", "/alkane-toolkit/") +PRIVATE = ("/alkane-sdk/", "/alkane-os/") class NoRedirect(HTTPRedirectHandler): diff --git a/ci/metrics.py b/ci/metrics.py index dadd2af..fc3a2f1 100644 --- a/ci/metrics.py +++ b/ci/metrics.py @@ -68,6 +68,18 @@ def svg(label: str, value: str, color: str) -> bytes: f'font-family="sans-serif" font-size="11">{value}\n').encode() +# Same bands as the toolkit's quality grade: coverage sets the grade. +GRADE_BANDS = ((80.0, "A", "#2ea043"), (60.0, "B", "#7cb342"), (40.0, "C", "#d4a72c"), + (20.0, "D", "#e16f24")) + + +def grade(coverage: float) -> tuple[str, str]: + for floor, letter, color in GRADE_BANDS: + if coverage >= floor: + return letter, color + return "F", "#cf222e" + + def generate() -> tuple[dict[str, bytes], dict]: tests, coverage = tests_and_coverage() sources = list((ROOT / "ci").glob("*.py")) @@ -76,14 +88,15 @@ def generate() -> tuple[dict[str, bytes], dict]: if line.strip() and not line.lstrip().startswith("#")) if "MIT License" not in (ROOT / "LICENSE").read_text(): raise RuntimeError("site license is not MIT") + letter, color = grade(coverage) values = { "tests": (str(tests), "#2ea043"), - "coverage": (f"{coverage:.1f}%", "#2ea043"), + "coverage": (f"{coverage:.1f}%", color), "bench": ("n/a", "#777"), "bench-coverage": ("n/a", "#777"), "loc": (compact_lines(lines), "#0969da"), "dependencies": ("0", "#2ea043"), - "quality": ("A", "#2ea043"), + "quality": (letter, color), "license": ("MIT", "#0969da"), } labels = {"bench-coverage": "bench coverage", "loc": "LOC", @@ -94,7 +107,7 @@ def generate() -> tuple[dict[str, bytes], dict]: "statement_coverage_percent": round(coverage, 1), "source_lines": lines, "dependencies": 0, "benchmarks": None, "benchmark_coverage_percent": None, - "quality": "A", "license": "MIT"} + "quality": letter, "license": "MIT"} return badges, metrics diff --git a/ci/test_validate_site.py b/ci/test_validate_site.py index 5d924a1..d1057e9 100644 --- a/ci/test_validate_site.py +++ b/ci/test_validate_site.py @@ -12,7 +12,7 @@ from validate_site import validate INDEX = """Alkane benchmarks

Alkane benchmarks

SDK -OSToolkit""" +OS""" PAGE = """Alkane benchmarks

Alkane benchmarks

10 ns/op
""" @@ -23,7 +23,7 @@ class SiteValidationTests(unittest.TestCase): self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) (self.root / "index.html").write_text(INDEX) - for repo in ("sdk", "os", "toolkit"): + for repo in ("sdk", "os"): (self.root / repo).mkdir() (self.root / repo / "index.html").write_text(PAGE) @@ -60,12 +60,17 @@ class SiteValidationTests(unittest.TestCase): ) self.assertTrue(any("external or active reference" in error for error in validate(self.root))) + def test_allows_link_to_toolkit_site(self): + (self.root / "index.html").write_text(INDEX.replace( + "", 'Toolkit')) + self.assertEqual(validate(self.root), []) + def test_rejects_extra_files(self): (self.root / "source.tar.gz").write_bytes(b"private source") self.assertTrue(any("unexpected published files" in error for error in validate(self.root))) - def test_requires_all_three_reports(self): - (self.root / "toolkit/index.html").unlink() + def test_requires_both_reports(self): + (self.root / "os/index.html").unlink() self.assertTrue(any("missing regular page" in error for error in validate(self.root))) def test_rejects_css_import(self): @@ -103,7 +108,7 @@ class SiteValidationTests(unittest.TestCase): self.assertTrue(any("contains a symlink" in error for error in validate(self.root))) def test_requires_index_navigation(self): - (self.root / "index.html").write_text(INDEX.replace('Toolkit', "")) + (self.root / "index.html").write_text(INDEX.replace('OS', "")) self.assertTrue(any("missing project links" in error for error in validate(self.root))) def test_requires_measurements_or_pending_notice(self): diff --git a/ci/validate_site.py b/ci/validate_site.py index f6a6947..7d3eb75 100644 --- a/ci/validate_site.py +++ b/ci/validate_site.py @@ -7,13 +7,15 @@ from pathlib import Path import re -PAGES = ("index.html", "sdk/index.html", "os/index.html", "toolkit/index.html") +PAGES = ("index.html", "sdk/index.html", "os/index.html") +# The toolkit is public and publishes its own site; the index links to it. +TOOLKIT_SITE = "https://blueflare.gitea.bluefla.re/alkane-toolkit/bench/" ALLOWED_TAGS = {"html", "head", "meta", "title", "style", "body", "h1", "p", "a", "ul", "li", "div", "small", "table", "thead", "tbody", "tr", "th", "td"} ALLOWED_ATTRIBUTES = {"html": {"lang"}, "a": {"href"}, "div": {"class"}, "th": {"scope"}} -LOCAL_LINKS = {"sdk/", "os/", "toolkit/", "../"} +LOCAL_LINKS = {"sdk/", "os/", "../", TOOLKIT_SITE} FORBIDDEN_TEXT = ("BEGIN PRIVATE KEY", "GITEA_TOKEN", "ghp_", "glpat-") COMMIT_LINK = re.compile( r"https://gitea\.bluefla\.re/blueflare/alkane-(?:sdk|os|toolkit)/commit/[0-9a-f]{40}" @@ -86,7 +88,7 @@ def validate_page(path: Path, relative: str) -> list[str]: if re.search(r"url\s*\(|@import", text, re.IGNORECASE): errors.append(f"{relative}: external CSS reference is not allowed") if relative == "index.html": - if not {"sdk/", "os/", "toolkit/"}.issubset(parser.links): + if not {"sdk/", "os/"}.issubset(parser.links): errors.append("index.html: missing project links") elif not parser.tables and "Waiting for a completed measured CI run." not in text: errors.append(f"{relative}: no measurements or pending notice") @@ -111,7 +113,7 @@ def validate(root: Path) -> list[str]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("site", type=Path) - parser.add_argument("--page", choices=("sdk", "os", "toolkit"), + parser.add_argument("--page", choices=("sdk", "os"), help="validate one generated report before publication") args = parser.parse_args() errors = (validate_page(args.site, f"{args.page}/index.html")