The toolkit repository is public and now publishes its own Pages site (benchmarks, badges, quality dashboard), so this site covers only the private SDK and OS. - validate_site: drop the toolkit page; the index may link to the toolkit's bench page. - check_live_pages: /alkane-toolkit/ and its bench page must be public; only the SDK and OS sites must stay closed. - metrics: grade coverage on the toolkit's bands instead of a fixed green A. - README: one CI badge plus the license; the script metrics no longer pose as SDK or OS metrics. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
140 lines
5.5 KiB
Python
140 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Measure site guard tests and publish honest repository badges."""
|
|
|
|
import argparse
|
|
import ast
|
|
import html
|
|
import json
|
|
from pathlib import Path
|
|
import trace
|
|
import unittest
|
|
|
|
from badge_store import publish_file
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OWNED = (ROOT / "ci/validate_site.py", ROOT / "ci/version.py")
|
|
|
|
|
|
def statement_lines(path: Path) -> set[int]:
|
|
tree = ast.parse(path.read_text())
|
|
return {node.lineno for node in ast.walk(tree)
|
|
if isinstance(node, ast.stmt)
|
|
and not isinstance(node, (ast.FunctionDef, ast.ClassDef,
|
|
ast.Import, ast.ImportFrom))
|
|
and not (isinstance(node, ast.Expr)
|
|
and isinstance(node.value, ast.Constant)
|
|
and isinstance(node.value.value, str))}
|
|
|
|
|
|
def tests_and_coverage() -> tuple[int, float]:
|
|
suite = unittest.TestLoader().discover(str(ROOT / "ci"), pattern="test_*.py")
|
|
count = suite.countTestCases()
|
|
tracer = trace.Trace(count=True, trace=False)
|
|
result = tracer.runfunc(unittest.TextTestRunner(verbosity=0).run, suite)
|
|
if not result.wasSuccessful():
|
|
raise RuntimeError("site guard tests failed")
|
|
hit = {(Path(path).resolve(), line) for (path, line), times
|
|
in tracer.results().counts.items() if times > 0}
|
|
executable = {(path, line) for path in OWNED for line in statement_lines(path)}
|
|
covered = sum(item in hit for item in executable)
|
|
return count, 100 * covered / len(executable)
|
|
|
|
|
|
def compact_lines(count: int) -> str:
|
|
if count < 1000:
|
|
return str(count)
|
|
for scale, suffix in ((1_000_000_000, "G"), (1_000_000, "M"), (1000, "K")):
|
|
if count >= scale:
|
|
number = count / scale
|
|
return f"{number:.1f}{suffix}" if number < 10 else f"{number:.0f}{suffix}"
|
|
raise AssertionError("unreachable")
|
|
|
|
|
|
def svg(label: str, value: str, color: str) -> bytes:
|
|
label = html.escape(label)
|
|
value = html.escape(value)
|
|
left = max(40, len(label) * 7 + 14)
|
|
right = max(36, len(value) * 7 + 14)
|
|
total = left + right
|
|
title = f"{label}: {value}"
|
|
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" '
|
|
f'role="img" aria-label="{title}"><title>{title}</title>'
|
|
f'<rect width="{total}" height="20" rx="3" fill="#555"/>'
|
|
f'<path d="M{left} 0h{right - 3}a3 3 0 0 1 3 3v14a3 3 0 0 1-3 3H{left}z" fill="{color}"/>'
|
|
f'<text x="{left // 2}" y="14" fill="#fff" text-anchor="middle" '
|
|
f'font-family="sans-serif" font-size="11">{label}</text>'
|
|
f'<text x="{left + right // 2}" y="14" fill="#fff" text-anchor="middle" '
|
|
f'font-family="sans-serif" font-size="11">{value}</text></svg>\n').encode()
|
|
|
|
|
|
# Same bands as the toolkit's quality grade: coverage sets the grade.
|
|
GRADE_BANDS = ((80.0, "A", "#2ea043"), (60.0, "B", "#7cb342"), (40.0, "C", "#d4a72c"),
|
|
(20.0, "D", "#e16f24"))
|
|
|
|
|
|
def grade(coverage: float) -> tuple[str, str]:
|
|
for floor, letter, color in GRADE_BANDS:
|
|
if coverage >= floor:
|
|
return letter, color
|
|
return "F", "#cf222e"
|
|
|
|
|
|
def generate() -> tuple[dict[str, bytes], dict]:
|
|
tests, coverage = tests_and_coverage()
|
|
sources = list((ROOT / "ci").glob("*.py"))
|
|
lines = sum(1 for path in sources if not path.name.startswith("test_")
|
|
for line in path.read_text().splitlines()
|
|
if line.strip() and not line.lstrip().startswith("#"))
|
|
if "MIT License" not in (ROOT / "LICENSE").read_text():
|
|
raise RuntimeError("site license is not MIT")
|
|
letter, color = grade(coverage)
|
|
values = {
|
|
"tests": (str(tests), "#2ea043"),
|
|
"coverage": (f"{coverage:.1f}%", color),
|
|
"bench": ("n/a", "#777"),
|
|
"bench-coverage": ("n/a", "#777"),
|
|
"loc": (compact_lines(lines), "#0969da"),
|
|
"dependencies": ("0", "#2ea043"),
|
|
"quality": (letter, color),
|
|
"license": ("MIT", "#0969da"),
|
|
}
|
|
labels = {"bench-coverage": "bench coverage", "loc": "LOC",
|
|
"quality": "code quality"}
|
|
badges = {name: svg(labels.get(name, name), value, color)
|
|
for name, (value, color) in values.items()}
|
|
metrics = {"schema": 1, "test_cases": tests,
|
|
"statement_coverage_percent": round(coverage, 1),
|
|
"source_lines": lines, "dependencies": 0,
|
|
"benchmarks": None, "benchmark_coverage_percent": None,
|
|
"quality": letter, "license": "MIT"}
|
|
return badges, metrics
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--preview", type=Path)
|
|
args = parser.parse_args()
|
|
badges, metrics = generate()
|
|
data = json.dumps(metrics, indent=2).encode() + b"\n"
|
|
if args.preview:
|
|
args.preview.mkdir(parents=True, exist_ok=True)
|
|
for name, content in badges.items():
|
|
(args.preview / f"{name}.svg").write_bytes(content)
|
|
(args.preview / "metrics.json").write_bytes(data)
|
|
else:
|
|
import os
|
|
api = os.environ["GITHUB_API_URL"].rstrip("/")
|
|
repo = os.environ["GITHUB_REPOSITORY"]
|
|
token = os.environ["GITEA_TOKEN"]
|
|
for name, content in badges.items():
|
|
publish_file(api, repo, f"badges/{name}.svg", content, token,
|
|
f"Measure {name} badge")
|
|
publish_file(api, repo, "badges/metrics.json", data, token,
|
|
"Measure site repository")
|
|
print(f"Measured {metrics['test_cases']} tests and {metrics['statement_coverage_percent']}% statement coverage")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|