Separate research evidence and speed scoped OS PR checks
source-pr / source (pull_request) Successful in 10s
source-pr / source (pull_request) Successful in 10s
This commit is contained in:
@@ -36,6 +36,11 @@ jobs:
|
||||
run: python3 ci/test-metrics.py
|
||||
- name: Release evidence gate checks
|
||||
run: python3 ci/test-conformance.py
|
||||
- name: Research and experiment report checks
|
||||
run: |
|
||||
python3 ci/test-evidence-status.py
|
||||
python3 ci/test-experiment-priority.py
|
||||
python3 ci/test-pr-surface.py
|
||||
- name: Release publication checks
|
||||
run: python3 ci/test-release.py
|
||||
- name: Release version checks
|
||||
@@ -101,6 +106,12 @@ jobs:
|
||||
with:
|
||||
name: alkane-os-benchmark-site-preview
|
||||
path: bench-site/index.html
|
||||
- name: Separate research from release evidence
|
||||
run: python3 ci/evidence_status.py --output evidence-status-current.json
|
||||
- uses: https://github.com/ChristopherHX/gitea-upload-artifact@81f940d004763f986ba3582c007fd842dd5cb0d7
|
||||
with:
|
||||
name: alkane-os-evidence-status
|
||||
path: evidence-status-current.json
|
||||
- name: Pairing and simulator conformance
|
||||
run: python3 ci/check-conformance.py
|
||||
- name: Select release version
|
||||
|
||||
@@ -18,14 +18,40 @@ jobs:
|
||||
CARGO_TARGET_DIR: target
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Detect changed surface
|
||||
id: surface
|
||||
run: python3 ci/pr_surface.py >> "$GITHUB_OUTPUT"
|
||||
- name: Check patch whitespace
|
||||
run: |
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = pull_request ]; then
|
||||
base="$(git merge-base HEAD "origin/$GITHUB_BASE_REF")"
|
||||
git diff --check "$base" HEAD
|
||||
else
|
||||
git diff --check HEAD^ HEAD
|
||||
fi
|
||||
- name: Check CI and evidence tools
|
||||
if: steps.surface.outputs.surface != 'documentation'
|
||||
run: |
|
||||
python3 -m compileall -q ci
|
||||
for test in ci/test-*.py; do python3 "$test"; done
|
||||
python3 ci/evidence_status.py
|
||||
python3 ci/experiment_priority.py
|
||||
- name: Install source formatter
|
||||
if: steps.surface.outputs.surface == 'rust'
|
||||
run: rustup component add rustfmt
|
||||
- name: Check source formatting
|
||||
if: steps.surface.outputs.surface == 'rust'
|
||||
run: cargo fmt --check
|
||||
- name: Check release numbering
|
||||
run: python3 ci/test-version.py
|
||||
- name: Build source
|
||||
run: cargo build --release --offline --locked
|
||||
- name: Check changed Rust source
|
||||
if: steps.surface.outputs.surface == 'rust'
|
||||
run: |
|
||||
if [ "${GITHUB_BASE_REF:-}" = main ]; then
|
||||
cargo build --release --offline --locked
|
||||
else
|
||||
cargo check --offline --locked
|
||||
fi
|
||||
- name: Clean generated build
|
||||
if: always()
|
||||
run: python3 ci/cleanup-generated.py
|
||||
|
||||
@@ -5,6 +5,13 @@ currently fails because the simulator and bench evidence producer is not yet
|
||||
implemented. A green Rust build, host HAL replay probe, or rootfs archive does
|
||||
not fill any cell.
|
||||
|
||||
`research-evidence.json` records scoped historical observations with source
|
||||
hashes. `evidence_status.py` reports those separately from declarations in
|
||||
`conformance.json`. Neither research claims nor manifest declarations count
|
||||
as verified release cells. The current-run checker below is the only release
|
||||
validator. `experiment-queue.json` ranks the next protocol and controller
|
||||
experiments by the number of affected cells; blocked work stays visible.
|
||||
|
||||
Each CI run must put a JSON report at
|
||||
`ci/evidence/<GITHUB_RUN_ID>/<board>/<chip>.json`. The report binds its
|
||||
`run_id`, `source_commit`, `os_binary_sha256`, `board`, and `chip` to the
|
||||
|
||||
@@ -10,7 +10,8 @@ import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIRECTORIES = ("target", "coverage", "dist", "bench-site")
|
||||
FILES = ("tests-current.txt", "bench-current.txt", "quality-current.json")
|
||||
FILES = ("tests-current.txt", "bench-current.txt", "quality-current.json",
|
||||
"evidence-status-current.json")
|
||||
|
||||
|
||||
def cleanup() -> None:
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Summarize research separately from current-run release conformance."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
|
||||
LEVELS = {"catalogued", "protocol_decoded"}
|
||||
DIGEST = re.compile(r"[0-9a-f]{64}\Z")
|
||||
|
||||
|
||||
def summarize(matrix: dict, research: dict) -> dict:
|
||||
if matrix.get("schema") != 1 or research.get("schema") != 1:
|
||||
raise ValueError("unsupported evidence schema")
|
||||
boards = matrix["boards"]
|
||||
chips = matrix["chips"]
|
||||
vendors = matrix["vendors"]
|
||||
expected = {(board, chip) for board in boards for chip in chips
|
||||
if vendors[board] == vendors[chip]}
|
||||
rows = matrix["targets"]
|
||||
actual = [(row["board"], row["chip"]) for row in rows]
|
||||
if set(actual) != expected or len(actual) != len(expected) or len(actual) != 64:
|
||||
raise ValueError("release target matrix is incomplete or duplicated")
|
||||
levels = {"catalogued": set(), "protocol_decoded": set()}
|
||||
for claim in research["claims"]:
|
||||
cell = claim["board"], claim["chip"]
|
||||
level = claim["level"]
|
||||
if cell not in expected or level not in LEVELS:
|
||||
raise ValueError("research claim is outside target matrix or level")
|
||||
if not isinstance(claim.get("scope"), str) or not claim["scope"].strip():
|
||||
raise ValueError("research claim lacks scope")
|
||||
if not isinstance(claim.get("source"), str) or not claim["source"].strip():
|
||||
raise ValueError("research claim lacks provenance")
|
||||
if not isinstance(claim.get("sha256"), str) or not DIGEST.fullmatch(claim["sha256"]):
|
||||
raise ValueError("research claim lacks a source digest")
|
||||
levels[level].add(cell)
|
||||
declared = {
|
||||
"stock_result": sum(row.get("stock_result") in {"mines", "rejects"} for row in rows),
|
||||
"alkane_share_capture": sum(bool(row.get("alkane_share_accept_capture")) for row in rows),
|
||||
"bench_validation": sum(bool(row.get("bench_validation")) for row in rows),
|
||||
}
|
||||
return {
|
||||
"schema": 1,
|
||||
"target_cells": len(expected),
|
||||
"research_only": {key: len(value) for key, value in levels.items()},
|
||||
"release_manifest_declarations_unverified": declared,
|
||||
"verified_release_cells": None,
|
||||
"note": "Only check-conformance.py may verify current-run artifacts; research claims never satisfy a release cell",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--matrix", type=Path, default=Path("ci/conformance.json"))
|
||||
parser.add_argument("--research", type=Path, default=Path("ci/research-evidence.json"))
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
report = summarize(json.loads(args.matrix.read_text()), json.loads(args.research.read_text()))
|
||||
output = json.dumps(report, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
args.output.write_text(output)
|
||||
else:
|
||||
print(output, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"experiments": [
|
||||
{
|
||||
"id": "am335x-original-nand-boot",
|
||||
"kind": "controller",
|
||||
"board": "am335x-bb",
|
||||
"chip": null,
|
||||
"psu": null,
|
||||
"state": "ready_local",
|
||||
"next_evidence": "Boot hash-pinned stock NAND through the original ROM and SPL path; record the first failing register",
|
||||
"blocker": null
|
||||
},
|
||||
{
|
||||
"id": "bm1368-command-and-nonce",
|
||||
"kind": "chip",
|
||||
"board": null,
|
||||
"chip": "BM1368",
|
||||
"psu": null,
|
||||
"state": "needs_machine",
|
||||
"next_evidence": "Capture real BM1368 enumeration, job, and nonce frames on a recovered test unit",
|
||||
"blocker": "No verified BM1368 silicon candidate in the current inventory"
|
||||
},
|
||||
{
|
||||
"id": "zynq-bm1366-fpga-window",
|
||||
"kind": "transport",
|
||||
"board": "zynq-7010",
|
||||
"chip": "BM1366",
|
||||
"psu": "APW121215d/e",
|
||||
"state": "needs_identity",
|
||||
"next_evidence": "Read a bounded FPGA register/FIFO trace on a verified rooted test unit",
|
||||
"blocker": "Historical mule 10.1.2.137 now reports BTM_S21PRO; do not infer BM1366 from its address"
|
||||
},
|
||||
{
|
||||
"id": "a113d-bm1370-apw17-voltage",
|
||||
"kind": "psu",
|
||||
"board": "a113d",
|
||||
"chip": "BM1370",
|
||||
"psu": "APW171215a/c",
|
||||
"state": "needs_identity",
|
||||
"next_evidence": "Decode captured GPIO bit-bang command and read-only voltage response against an independent meter",
|
||||
"blocker": "Historical 10.4.0.192 unit is not currently reachable for confirmation"
|
||||
},
|
||||
{
|
||||
"id": "h6-kf1950-power-and-asic",
|
||||
"kind": "chip_psu",
|
||||
"board": "allwinner-h6",
|
||||
"chip": "KF1950",
|
||||
"psu": "unknown",
|
||||
"state": "needs_machine",
|
||||
"next_evidence": "Observe actual ASIC and PSU bus frames on a matched miner before modeling either device",
|
||||
"blocker": "Current H6 bench board is bare and its recovery RX path is unproven"
|
||||
},
|
||||
{
|
||||
"id": "s19-88-chain-enumeration",
|
||||
"kind": "topology_psu",
|
||||
"board": "a113d",
|
||||
"chip": "BM1362",
|
||||
"psu": "APW121417b",
|
||||
"state": "needs_identity",
|
||||
"next_evidence": "Capture the 88-chip address walk and read-only PSU identity on a rooted matched test unit",
|
||||
"blocker": "10.7.1.190 reports BTM_S19_88, but its SSH key and physical recovery are unverified"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Rank representative experiments without changing the 64-cell release gate."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
STATES = {"ready_local", "needs_identity", "needs_machine"}
|
||||
|
||||
|
||||
def rank(matrix: dict, queue: dict) -> list[dict]:
|
||||
if queue.get("schema") != 1 or matrix.get("schema") != 1:
|
||||
raise ValueError("unsupported experiment schema")
|
||||
targets = {(row["board"], row["chip"]) for row in matrix["targets"]}
|
||||
if len(targets) != 64 or len(matrix["targets"]) != 64:
|
||||
raise ValueError("expected 64 unique release targets")
|
||||
seen = set()
|
||||
result = []
|
||||
for experiment in queue["experiments"]:
|
||||
ident = experiment["id"]
|
||||
if ident in seen:
|
||||
raise ValueError("duplicate experiment id")
|
||||
seen.add(ident)
|
||||
state = experiment["state"]
|
||||
if state not in STATES:
|
||||
raise ValueError("unknown experiment state")
|
||||
board, chip = experiment["board"], experiment["chip"]
|
||||
if board is None and chip is None:
|
||||
raise ValueError("experiment has no target selector")
|
||||
affected = sorted((b, c) for b, c in targets
|
||||
if (board is None or board == b) and (chip is None or chip == c))
|
||||
if not affected:
|
||||
raise ValueError("experiment selects no same-vendor target")
|
||||
if not experiment.get("next_evidence"):
|
||||
raise ValueError("experiment lacks a measurable next result")
|
||||
if state != "ready_local" and not experiment.get("blocker"):
|
||||
raise ValueError("blocked experiment lacks a blocker")
|
||||
result.append({"id": ident, "kind": experiment["kind"], "state": state,
|
||||
"affected_release_cells": len(affected), "board": board,
|
||||
"chip": chip, "psu": experiment["psu"],
|
||||
"next_evidence": experiment["next_evidence"],
|
||||
"blocker": experiment["blocker"]})
|
||||
return sorted(result, key=lambda row: (row["state"] != "ready_local",
|
||||
-row["affected_release_cells"], row["id"]))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--matrix", type=Path, default=Path("ci/conformance.json"))
|
||||
parser.add_argument("--queue", type=Path, default=Path("ci/experiment-queue.json"))
|
||||
args = parser.parse_args()
|
||||
for item in rank(json.loads(args.matrix.read_text()), json.loads(args.queue.read_text())):
|
||||
print(f"{item['id']}: {item['state']}, {item['affected_release_cells']} cells")
|
||||
print(f" next: {item['next_evidence']}")
|
||||
if item["blocker"]:
|
||||
print(f" blocked: {item['blocker']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Select a safe fast PR check from the exact merge-base diff."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def classify(paths: list[str]) -> str:
|
||||
if not paths:
|
||||
return "rust"
|
||||
docs = {"README.md", "LICENSE", "AGENTS.md", "ci/CONFORMANCE.md"}
|
||||
def is_doc(path: str) -> bool:
|
||||
return path in docs or path.startswith("docs/") and path.endswith(".md")
|
||||
def is_tool(path: str) -> bool:
|
||||
return (path.startswith("ci/") and Path(path).suffix in {".py", ".json"}
|
||||
or path.startswith(".gitea/workflows/") and path.endswith(".yml"))
|
||||
if all(is_doc(path) for path in paths):
|
||||
return "documentation"
|
||||
if all(is_doc(path) or is_tool(path) for path in paths):
|
||||
return "tooling"
|
||||
return "rust"
|
||||
|
||||
|
||||
def changed_paths() -> list[str]:
|
||||
if os.environ.get("GITHUB_EVENT_NAME") != "pull_request":
|
||||
return []
|
||||
base = os.environ.get("GITHUB_BASE_REF")
|
||||
if base not in {"next", "main"}:
|
||||
return []
|
||||
try:
|
||||
merge_base = subprocess.check_output(
|
||||
["git", "merge-base", "HEAD", f"origin/{base}"], cwd=ROOT).strip()
|
||||
diff = subprocess.check_output(
|
||||
["git", "diff", "--name-only", "-z", merge_base, "HEAD"], cwd=ROOT)
|
||||
return [path.decode() for path in diff.split(b"\0") if path]
|
||||
except (OSError, UnicodeError, subprocess.CalledProcessError):
|
||||
return []
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"surface={classify(changed_paths())}\n")
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"claims": [
|
||||
{
|
||||
"board": "a113d",
|
||||
"chip": "BM1366",
|
||||
"level": "protocol_decoded",
|
||||
"source": "miner-firmware/firmware-analysis/prong-b/aml-112/fullcov/capC1.strace",
|
||||
"sha256": "05303e93f6d00433a6db5a9ee973b6cdb0351f3efa35d293dd172ede7da89d54",
|
||||
"scope": "Historical BM1366 UART enumeration and nonce frames on an A113D S19k Pro"
|
||||
},
|
||||
{
|
||||
"board": "zynq-7010",
|
||||
"chip": "BM1366",
|
||||
"level": "catalogued",
|
||||
"source": "miner-firmware/firmware-analysis/prong-b/xil-137/fullcov/capC1.strace",
|
||||
"sha256": "3daa3f7381580ef87f3b5925d5a03fdb4d2e7cd93a82b943ef2578da3d545607",
|
||||
"scope": "Historical sensor and board observations; FPGA chip data path was not captured"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Research observations cannot promote a target into release conformance."""
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
spec = importlib.util.spec_from_file_location("evidence_status", ROOT / "evidence_status.py")
|
||||
status = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(status)
|
||||
MATRIX = json.loads((ROOT / "conformance.json").read_text())
|
||||
RESEARCH = json.loads((ROOT / "research-evidence.json").read_text())
|
||||
|
||||
|
||||
class EvidenceStatusTest(unittest.TestCase):
|
||||
def test_historical_protocol_does_not_satisfy_release(self):
|
||||
report = status.summarize(MATRIX, RESEARCH)
|
||||
self.assertEqual(report["target_cells"], 64)
|
||||
self.assertEqual(report["research_only"], {"catalogued": 1, "protocol_decoded": 1})
|
||||
self.assertIsNone(report["verified_release_cells"])
|
||||
self.assertEqual(sum(report["release_manifest_declarations_unverified"].values()), 0)
|
||||
|
||||
def test_rejects_missing_cell_and_unsupported_claim(self):
|
||||
matrix = copy.deepcopy(MATRIX)
|
||||
matrix["targets"].pop()
|
||||
with self.assertRaisesRegex(ValueError, "incomplete"):
|
||||
status.summarize(matrix, RESEARCH)
|
||||
research = copy.deepcopy(RESEARCH)
|
||||
research["claims"][0]["level"] = "pool_accept"
|
||||
with self.assertRaisesRegex(ValueError, "outside target matrix or level"):
|
||||
status.summarize(MATRIX, research)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Representative experiments preserve the full release target set."""
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
spec = importlib.util.spec_from_file_location("experiment_priority", ROOT / "experiment_priority.py")
|
||||
priority = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(priority)
|
||||
MATRIX = json.loads((ROOT / "conformance.json").read_text())
|
||||
QUEUE = json.loads((ROOT / "experiment-queue.json").read_text())
|
||||
|
||||
|
||||
class ExperimentPriorityTest(unittest.TestCase):
|
||||
def test_local_boot_model_ranks_first_without_reducing_gate(self):
|
||||
ordered = priority.rank(MATRIX, QUEUE)
|
||||
self.assertEqual(ordered[0]["id"], "am335x-original-nand-boot")
|
||||
self.assertEqual(ordered[0]["affected_release_cells"], 13)
|
||||
self.assertEqual(len(MATRIX["targets"]), 64)
|
||||
self.assertEqual(next(row for row in ordered if row["id"] == "bm1368-command-and-nonce")
|
||||
["affected_release_cells"], 4)
|
||||
|
||||
def test_no_selector_or_unexplained_blocker(self):
|
||||
queue = copy.deepcopy(QUEUE)
|
||||
queue["experiments"][0]["board"] = None
|
||||
with self.assertRaisesRegex(ValueError, "no target selector"):
|
||||
priority.rank(MATRIX, queue)
|
||||
queue = copy.deepcopy(QUEUE)
|
||||
queue["experiments"][1]["blocker"] = None
|
||||
with self.assertRaisesRegex(ValueError, "lacks a blocker"):
|
||||
priority.rank(MATRIX, queue)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""Unknown PR paths must retain compiled source checks."""
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
spec = importlib.util.spec_from_file_location("pr_surface", Path(__file__).with_name("pr_surface.py"))
|
||||
surface = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(surface)
|
||||
|
||||
|
||||
class PrSurfaceTest(unittest.TestCase):
|
||||
def test_documentation_and_tooling(self):
|
||||
self.assertEqual(surface.classify(["README.md", "docs/boards.md"]), "documentation")
|
||||
self.assertEqual(surface.classify(["ci/evidence_status.py", "ci/CONFORMANCE.md"]), "tooling")
|
||||
|
||||
def test_unknown_and_rust_paths_fail_open_to_build(self):
|
||||
for paths in ([], ["src/main.rs"], ["Cargo.toml"], ["unknown/file"],
|
||||
["ci/evidence_status.py", "src/main.rs"]):
|
||||
with self.subTest(paths=paths):
|
||||
self.assertEqual(surface.classify(paths), "rust")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user