Files
alkane-benchmarks/ci/metrics.py
T
noahandClaude Opus 5.5 35b96c493f Hand the toolkit benchmarks to the toolkit's own site
The toolkit repository is public and now publishes its own Pages site
(benchmarks, badges, quality dashboard), so this site covers only the
private SDK and OS.

- validate_site: drop the toolkit page; the index may link to the
  toolkit's bench page.
- check_live_pages: /alkane-toolkit/ and its bench page must be public;
  only the SDK and OS sites must stay closed.
- metrics: grade coverage on the toolkit's bands instead of a fixed
  green A.
- README: one CI badge plus the license; the script metrics no longer
  pose as SDK or OS metrics.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 11:03:55 -06:00

140 lines
5.5 KiB
Python

#!/usr/bin/env python3
"""Measure site guard tests and publish honest repository badges."""
import argparse
import ast
import html
import json
from pathlib import Path
import trace
import unittest
from badge_store import publish_file
ROOT = Path(__file__).resolve().parents[1]
OWNED = (ROOT / "ci/validate_site.py", ROOT / "ci/version.py")
def statement_lines(path: Path) -> set[int]:
tree = ast.parse(path.read_text())
return {node.lineno for node in ast.walk(tree)
if isinstance(node, ast.stmt)
and not isinstance(node, (ast.FunctionDef, ast.ClassDef,
ast.Import, ast.ImportFrom))
and not (isinstance(node, ast.Expr)
and isinstance(node.value, ast.Constant)
and isinstance(node.value.value, str))}
def tests_and_coverage() -> tuple[int, float]:
suite = unittest.TestLoader().discover(str(ROOT / "ci"), pattern="test_*.py")
count = suite.countTestCases()
tracer = trace.Trace(count=True, trace=False)
result = tracer.runfunc(unittest.TextTestRunner(verbosity=0).run, suite)
if not result.wasSuccessful():
raise RuntimeError("site guard tests failed")
hit = {(Path(path).resolve(), line) for (path, line), times
in tracer.results().counts.items() if times > 0}
executable = {(path, line) for path in OWNED for line in statement_lines(path)}
covered = sum(item in hit for item in executable)
return count, 100 * covered / len(executable)
def compact_lines(count: int) -> str:
if count < 1000:
return str(count)
for scale, suffix in ((1_000_000_000, "G"), (1_000_000, "M"), (1000, "K")):
if count >= scale:
number = count / scale
return f"{number:.1f}{suffix}" if number < 10 else f"{number:.0f}{suffix}"
raise AssertionError("unreachable")
def svg(label: str, value: str, color: str) -> bytes:
label = html.escape(label)
value = html.escape(value)
left = max(40, len(label) * 7 + 14)
right = max(36, len(value) * 7 + 14)
total = left + right
title = f"{label}: {value}"
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" '
f'role="img" aria-label="{title}"><title>{title}</title>'
f'<rect width="{total}" height="20" rx="3" fill="#555"/>'
f'<path d="M{left} 0h{right - 3}a3 3 0 0 1 3 3v14a3 3 0 0 1-3 3H{left}z" fill="{color}"/>'
f'<text x="{left // 2}" y="14" fill="#fff" text-anchor="middle" '
f'font-family="sans-serif" font-size="11">{label}</text>'
f'<text x="{left + right // 2}" y="14" fill="#fff" text-anchor="middle" '
f'font-family="sans-serif" font-size="11">{value}</text></svg>\n').encode()
# Same bands as the toolkit's quality grade: coverage sets the grade.
GRADE_BANDS = ((80.0, "A", "#2ea043"), (60.0, "B", "#7cb342"), (40.0, "C", "#d4a72c"),
(20.0, "D", "#e16f24"))
def grade(coverage: float) -> tuple[str, str]:
for floor, letter, color in GRADE_BANDS:
if coverage >= floor:
return letter, color
return "F", "#cf222e"
def generate() -> tuple[dict[str, bytes], dict]:
tests, coverage = tests_and_coverage()
sources = list((ROOT / "ci").glob("*.py"))
lines = sum(1 for path in sources if not path.name.startswith("test_")
for line in path.read_text().splitlines()
if line.strip() and not line.lstrip().startswith("#"))
if "MIT License" not in (ROOT / "LICENSE").read_text():
raise RuntimeError("site license is not MIT")
letter, color = grade(coverage)
values = {
"tests": (str(tests), "#2ea043"),
"coverage": (f"{coverage:.1f}%", color),
"bench": ("n/a", "#777"),
"bench-coverage": ("n/a", "#777"),
"loc": (compact_lines(lines), "#0969da"),
"dependencies": ("0", "#2ea043"),
"quality": (letter, color),
"license": ("MIT", "#0969da"),
}
labels = {"bench-coverage": "bench coverage", "loc": "LOC",
"quality": "code quality"}
badges = {name: svg(labels.get(name, name), value, color)
for name, (value, color) in values.items()}
metrics = {"schema": 1, "test_cases": tests,
"statement_coverage_percent": round(coverage, 1),
"source_lines": lines, "dependencies": 0,
"benchmarks": None, "benchmark_coverage_percent": None,
"quality": letter, "license": "MIT"}
return badges, metrics
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--preview", type=Path)
args = parser.parse_args()
badges, metrics = generate()
data = json.dumps(metrics, indent=2).encode() + b"\n"
if args.preview:
args.preview.mkdir(parents=True, exist_ok=True)
for name, content in badges.items():
(args.preview / f"{name}.svg").write_bytes(content)
(args.preview / "metrics.json").write_bytes(data)
else:
import os
api = os.environ["GITHUB_API_URL"].rstrip("/")
repo = os.environ["GITHUB_REPOSITORY"]
token = os.environ["GITEA_TOKEN"]
for name, content in badges.items():
publish_file(api, repo, f"badges/{name}.svg", content, token,
f"Measure {name} badge")
publish_file(api, repo, "badges/metrics.json", data, token,
"Measure site repository")
print(f"Measured {metrics['test_cases']} tests and {metrics['statement_coverage_percent']}% statement coverage")
if __name__ == "__main__":
main()