Merge pull request 'Hand the toolkit benchmarks to the toolkit site; honest README badges' (#1) from toolkit-own-pages into main
main-site-gate / validate-and-release (push) Successful in 21s
main-site-gate / validate-and-release (push) Successful in 21s
This commit was merged in pull request #1.
This commit is contained in:
@@ -1,23 +1,19 @@
|
|||||||
# Alkane benchmarks
|
# Alkane benchmarks
|
||||||
|
|
||||||
[](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions)
|
[](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions)
|
||||||
[](https://gitea.bluefla.re/blueflare/alkane-benchmarks/actions)
|
|
||||||

|
|
||||||

|
|
||||||

|
|
||||||

|
|
||||||

|
|
||||||

|
|
||||||

|
|
||||||

|

|
||||||
|
|
||||||
Public [benchmark history](https://blueflare.gitea.bluefla.re/alkane-benchmarks/)
|
Public [benchmark history](https://blueflare.gitea.bluefla.re/alkane-benchmarks/)
|
||||||
for Alkane SDK, OS, and Toolkit. Scheduled publishers copy measured host data
|
for the Alkane SDK and OS, whose source repositories are private. Publishers
|
||||||
after CI. Source repositories remain private. These results do not measure
|
copy measured host data after CI. These results do not measure miner boards
|
||||||
miner boards or qualify a release.
|
or qualify a release.
|
||||||
|
|
||||||
This site has no microbenchmarks of its own, so its bench badges read `n/a`.
|
The toolkit is public and publishes its own benchmarks, badges and quality
|
||||||
Its coverage badge measures the site validator and version selector.
|
dashboard at <https://blueflare.gitea.bluefla.re/alkane-toolkit/>.
|
||||||
|
|
||||||
|
This repository holds only the site validator and publishing scripts, so it
|
||||||
|
shows no test, coverage or size badges: those would describe the scripts, not
|
||||||
|
the SDK or OS.
|
||||||
|
|
||||||
## Branches
|
## Branches
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# SPDX-License-Identifier: MIT
|
# SPDX-License-Identifier: MIT
|
||||||
"""Check that benchmark Pages are public and source repository Pages stay closed."""
|
"""Check that benchmark Pages are public and private source repositories stay closed.
|
||||||
|
|
||||||
|
The toolkit repository is public and serves its own site, benchmarks included.
|
||||||
|
"""
|
||||||
|
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||||
@@ -11,9 +14,10 @@ PUBLIC = (
|
|||||||
"/alkane-benchmarks/",
|
"/alkane-benchmarks/",
|
||||||
"/alkane-benchmarks/sdk/",
|
"/alkane-benchmarks/sdk/",
|
||||||
"/alkane-benchmarks/os/",
|
"/alkane-benchmarks/os/",
|
||||||
"/alkane-benchmarks/toolkit/",
|
"/alkane-toolkit/",
|
||||||
|
"/alkane-toolkit/bench/",
|
||||||
)
|
)
|
||||||
PRIVATE = ("/alkane-sdk/", "/alkane-os/", "/alkane-toolkit/")
|
PRIVATE = ("/alkane-sdk/", "/alkane-os/")
|
||||||
|
|
||||||
|
|
||||||
class NoRedirect(HTTPRedirectHandler):
|
class NoRedirect(HTTPRedirectHandler):
|
||||||
|
|||||||
+16
-3
@@ -68,6 +68,18 @@ def svg(label: str, value: str, color: str) -> bytes:
|
|||||||
f'font-family="sans-serif" font-size="11">{value}</text></svg>\n').encode()
|
f'font-family="sans-serif" font-size="11">{value}</text></svg>\n').encode()
|
||||||
|
|
||||||
|
|
||||||
|
# Same bands as the toolkit's quality grade: coverage sets the grade.
|
||||||
|
GRADE_BANDS = ((80.0, "A", "#2ea043"), (60.0, "B", "#7cb342"), (40.0, "C", "#d4a72c"),
|
||||||
|
(20.0, "D", "#e16f24"))
|
||||||
|
|
||||||
|
|
||||||
|
def grade(coverage: float) -> tuple[str, str]:
|
||||||
|
for floor, letter, color in GRADE_BANDS:
|
||||||
|
if coverage >= floor:
|
||||||
|
return letter, color
|
||||||
|
return "F", "#cf222e"
|
||||||
|
|
||||||
|
|
||||||
def generate() -> tuple[dict[str, bytes], dict]:
|
def generate() -> tuple[dict[str, bytes], dict]:
|
||||||
tests, coverage = tests_and_coverage()
|
tests, coverage = tests_and_coverage()
|
||||||
sources = list((ROOT / "ci").glob("*.py"))
|
sources = list((ROOT / "ci").glob("*.py"))
|
||||||
@@ -76,14 +88,15 @@ def generate() -> tuple[dict[str, bytes], dict]:
|
|||||||
if line.strip() and not line.lstrip().startswith("#"))
|
if line.strip() and not line.lstrip().startswith("#"))
|
||||||
if "MIT License" not in (ROOT / "LICENSE").read_text():
|
if "MIT License" not in (ROOT / "LICENSE").read_text():
|
||||||
raise RuntimeError("site license is not MIT")
|
raise RuntimeError("site license is not MIT")
|
||||||
|
letter, color = grade(coverage)
|
||||||
values = {
|
values = {
|
||||||
"tests": (str(tests), "#2ea043"),
|
"tests": (str(tests), "#2ea043"),
|
||||||
"coverage": (f"{coverage:.1f}%", "#2ea043"),
|
"coverage": (f"{coverage:.1f}%", color),
|
||||||
"bench": ("n/a", "#777"),
|
"bench": ("n/a", "#777"),
|
||||||
"bench-coverage": ("n/a", "#777"),
|
"bench-coverage": ("n/a", "#777"),
|
||||||
"loc": (compact_lines(lines), "#0969da"),
|
"loc": (compact_lines(lines), "#0969da"),
|
||||||
"dependencies": ("0", "#2ea043"),
|
"dependencies": ("0", "#2ea043"),
|
||||||
"quality": ("A", "#2ea043"),
|
"quality": (letter, color),
|
||||||
"license": ("MIT", "#0969da"),
|
"license": ("MIT", "#0969da"),
|
||||||
}
|
}
|
||||||
labels = {"bench-coverage": "bench coverage", "loc": "LOC",
|
labels = {"bench-coverage": "bench coverage", "loc": "LOC",
|
||||||
@@ -94,7 +107,7 @@ def generate() -> tuple[dict[str, bytes], dict]:
|
|||||||
"statement_coverage_percent": round(coverage, 1),
|
"statement_coverage_percent": round(coverage, 1),
|
||||||
"source_lines": lines, "dependencies": 0,
|
"source_lines": lines, "dependencies": 0,
|
||||||
"benchmarks": None, "benchmark_coverage_percent": None,
|
"benchmarks": None, "benchmark_coverage_percent": None,
|
||||||
"quality": "A", "license": "MIT"}
|
"quality": letter, "license": "MIT"}
|
||||||
return badges, metrics
|
return badges, metrics
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from validate_site import validate
|
|||||||
|
|
||||||
INDEX = """<!doctype html><html><head><title>Alkane benchmarks</title></head>
|
INDEX = """<!doctype html><html><head><title>Alkane benchmarks</title></head>
|
||||||
<body><h1>Alkane benchmarks</h1><a href="sdk/">SDK</a>
|
<body><h1>Alkane benchmarks</h1><a href="sdk/">SDK</a>
|
||||||
<a href="os/">OS</a><a href="toolkit/">Toolkit</a></body></html>"""
|
<a href="os/">OS</a></body></html>"""
|
||||||
PAGE = """<!doctype html><html><head><title>Alkane benchmarks</title></head>
|
PAGE = """<!doctype html><html><head><title>Alkane benchmarks</title></head>
|
||||||
<body><h1>Alkane benchmarks</h1><table><tr><td>10 ns/op</td></tr></table></body></html>"""
|
<body><h1>Alkane benchmarks</h1><table><tr><td>10 ns/op</td></tr></table></body></html>"""
|
||||||
|
|
||||||
@@ -23,7 +23,7 @@ class SiteValidationTests(unittest.TestCase):
|
|||||||
self.addCleanup(self.temp.cleanup)
|
self.addCleanup(self.temp.cleanup)
|
||||||
self.root = Path(self.temp.name)
|
self.root = Path(self.temp.name)
|
||||||
(self.root / "index.html").write_text(INDEX)
|
(self.root / "index.html").write_text(INDEX)
|
||||||
for repo in ("sdk", "os", "toolkit"):
|
for repo in ("sdk", "os"):
|
||||||
(self.root / repo).mkdir()
|
(self.root / repo).mkdir()
|
||||||
(self.root / repo / "index.html").write_text(PAGE)
|
(self.root / repo / "index.html").write_text(PAGE)
|
||||||
|
|
||||||
@@ -60,12 +60,17 @@ class SiteValidationTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertTrue(any("external or active reference" in error for error in validate(self.root)))
|
self.assertTrue(any("external or active reference" in error for error in validate(self.root)))
|
||||||
|
|
||||||
|
def test_allows_link_to_toolkit_site(self):
|
||||||
|
(self.root / "index.html").write_text(INDEX.replace(
|
||||||
|
"</body>", '<a href="https://blueflare.gitea.bluefla.re/alkane-toolkit/bench/">Toolkit</a></body>'))
|
||||||
|
self.assertEqual(validate(self.root), [])
|
||||||
|
|
||||||
def test_rejects_extra_files(self):
|
def test_rejects_extra_files(self):
|
||||||
(self.root / "source.tar.gz").write_bytes(b"private source")
|
(self.root / "source.tar.gz").write_bytes(b"private source")
|
||||||
self.assertTrue(any("unexpected published files" in error for error in validate(self.root)))
|
self.assertTrue(any("unexpected published files" in error for error in validate(self.root)))
|
||||||
|
|
||||||
def test_requires_all_three_reports(self):
|
def test_requires_both_reports(self):
|
||||||
(self.root / "toolkit/index.html").unlink()
|
(self.root / "os/index.html").unlink()
|
||||||
self.assertTrue(any("missing regular page" in error for error in validate(self.root)))
|
self.assertTrue(any("missing regular page" in error for error in validate(self.root)))
|
||||||
|
|
||||||
def test_rejects_css_import(self):
|
def test_rejects_css_import(self):
|
||||||
@@ -103,7 +108,7 @@ class SiteValidationTests(unittest.TestCase):
|
|||||||
self.assertTrue(any("contains a symlink" in error for error in validate(self.root)))
|
self.assertTrue(any("contains a symlink" in error for error in validate(self.root)))
|
||||||
|
|
||||||
def test_requires_index_navigation(self):
|
def test_requires_index_navigation(self):
|
||||||
(self.root / "index.html").write_text(INDEX.replace('<a href="toolkit/">Toolkit</a>', ""))
|
(self.root / "index.html").write_text(INDEX.replace('<a href="os/">OS</a>', ""))
|
||||||
self.assertTrue(any("missing project links" in error for error in validate(self.root)))
|
self.assertTrue(any("missing project links" in error for error in validate(self.root)))
|
||||||
|
|
||||||
def test_requires_measurements_or_pending_notice(self):
|
def test_requires_measurements_or_pending_notice(self):
|
||||||
|
|||||||
+6
-4
@@ -7,13 +7,15 @@ from pathlib import Path
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
|
|
||||||
PAGES = ("index.html", "sdk/index.html", "os/index.html", "toolkit/index.html")
|
PAGES = ("index.html", "sdk/index.html", "os/index.html")
|
||||||
|
# The toolkit is public and publishes its own site; the index links to it.
|
||||||
|
TOOLKIT_SITE = "https://blueflare.gitea.bluefla.re/alkane-toolkit/bench/"
|
||||||
ALLOWED_TAGS = {"html", "head", "meta", "title", "style", "body", "h1",
|
ALLOWED_TAGS = {"html", "head", "meta", "title", "style", "body", "h1",
|
||||||
"p", "a", "ul", "li", "div", "small", "table", "thead",
|
"p", "a", "ul", "li", "div", "small", "table", "thead",
|
||||||
"tbody", "tr", "th", "td"}
|
"tbody", "tr", "th", "td"}
|
||||||
ALLOWED_ATTRIBUTES = {"html": {"lang"}, "a": {"href"},
|
ALLOWED_ATTRIBUTES = {"html": {"lang"}, "a": {"href"},
|
||||||
"div": {"class"}, "th": {"scope"}}
|
"div": {"class"}, "th": {"scope"}}
|
||||||
LOCAL_LINKS = {"sdk/", "os/", "toolkit/", "../"}
|
LOCAL_LINKS = {"sdk/", "os/", "../", TOOLKIT_SITE}
|
||||||
FORBIDDEN_TEXT = ("BEGIN PRIVATE KEY", "GITEA_TOKEN", "ghp_", "glpat-")
|
FORBIDDEN_TEXT = ("BEGIN PRIVATE KEY", "GITEA_TOKEN", "ghp_", "glpat-")
|
||||||
COMMIT_LINK = re.compile(
|
COMMIT_LINK = re.compile(
|
||||||
r"https://gitea\.bluefla\.re/blueflare/alkane-(?:sdk|os|toolkit)/commit/[0-9a-f]{40}"
|
r"https://gitea\.bluefla\.re/blueflare/alkane-(?:sdk|os|toolkit)/commit/[0-9a-f]{40}"
|
||||||
@@ -86,7 +88,7 @@ def validate_page(path: Path, relative: str) -> list[str]:
|
|||||||
if re.search(r"url\s*\(|@import", text, re.IGNORECASE):
|
if re.search(r"url\s*\(|@import", text, re.IGNORECASE):
|
||||||
errors.append(f"{relative}: external CSS reference is not allowed")
|
errors.append(f"{relative}: external CSS reference is not allowed")
|
||||||
if relative == "index.html":
|
if relative == "index.html":
|
||||||
if not {"sdk/", "os/", "toolkit/"}.issubset(parser.links):
|
if not {"sdk/", "os/"}.issubset(parser.links):
|
||||||
errors.append("index.html: missing project links")
|
errors.append("index.html: missing project links")
|
||||||
elif not parser.tables and "Waiting for a completed measured CI run." not in text:
|
elif not parser.tables and "Waiting for a completed measured CI run." not in text:
|
||||||
errors.append(f"{relative}: no measurements or pending notice")
|
errors.append(f"{relative}: no measurements or pending notice")
|
||||||
@@ -111,7 +113,7 @@ def validate(root: Path) -> list[str]:
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("site", type=Path)
|
parser.add_argument("site", type=Path)
|
||||||
parser.add_argument("--page", choices=("sdk", "os", "toolkit"),
|
parser.add_argument("--page", choices=("sdk", "os"),
|
||||||
help="validate one generated report before publication")
|
help="validate one generated report before publication")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
errors = (validate_page(args.site, f"{args.page}/index.html")
|
errors = (validate_page(args.site, f"{args.page}/index.html")
|
||||||
|
|||||||
Reference in New Issue
Block a user