Files
alkane-os/ci/check-conformance.py
noah f6889940c3
main-release-gate / verify-and-release (push) Failing after 3m22s
Verify Litecoin Scrypt shares in conformance gate
2026-10-15 21:04:52 -06:00

402 lines
20 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Require the binary target matrix and traceable per-cell release evidence."""
import argparse
from decimal import Decimal, InvalidOperation
import hashlib
import json
import os
import pathlib
import re
import subprocess
import sys
REQUIRED_BOARDS = {
"am335x-bb": "bitmain",
"zynq-7010": "bitmain",
"cv1835": "bitmain",
"a113d": "bitmain",
"allwinner-h3": "whatsminer",
"allwinner-h6": "whatsminer",
"allwinner-h616": "whatsminer",
}
REQUIRED_CHIPS = {
**{chip: "bitmain" for chip in (
"BM1384", "BM1387", "BM1360", "BM1362", "BM1366", "BM1368",
"BM1370", "BM1396", "BM1397", "BM1398", "BM1489", "BM1491", "BM1746",
)},
**{chip: "whatsminer" for chip in (
"KF1950", "ID-1968", "ID-1980", "ID-1990",
)},
}
REQUIRED_ALGORITHMS = {
**{chip: "Sha256" for chip in REQUIRED_CHIPS if chip not in
("BM1489", "BM1491", "BM1746")},
"BM1489": "Scrypt", "BM1491": "Scrypt", "BM1746": "Equihash",
}
DIGEST = re.compile(r"[0-9a-f]{64}\Z")
COMMIT = re.compile(r"[0-9a-f]{40}\Z")
DIFF_ONE_TARGET = 0xffff << (8 * (0x1d - 3))
MAX_REPORT_BYTES = 1_048_576
def sha256_file(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1_048_576), b""):
digest.update(chunk)
return digest.hexdigest()
def load_report(path: pathlib.Path) -> dict:
if path.stat().st_size > MAX_REPORT_BYTES:
raise ValueError("evidence report exceeds 1 MiB")
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise ValueError("evidence report is not an object")
return value
def artifact(root: pathlib.Path, value: object) -> pathlib.Path:
if not isinstance(value, dict) or not isinstance(value.get("path"), str):
raise ValueError("artifact reference is missing")
digest = value.get("sha256")
if not isinstance(digest, str) or not DIGEST.fullmatch(digest):
raise ValueError("artifact digest is invalid")
path = (root / value["path"]).resolve()
if not path.is_relative_to(root.resolve()) or not path.is_file():
raise ValueError("artifact is missing or outside evidence directory")
if sha256_file(path) != digest:
raise ValueError("artifact digest mismatch")
return path
def hex_field(value: object, size: int | None = None) -> bytes:
if (not isinstance(value, str) or len(value) & 1
or len(value) > 131_072 or not re.fullmatch(r"[0-9a-fA-F]*", value)
or (size is not None and len(value) != size * 2)):
raise ValueError("invalid Stratum hex field")
return bytes.fromhex(value)
def u32_hex(value: object) -> int:
return int.from_bytes(hex_field(value, 4), "big")
def reconstructed_header(capture: dict, params: list[str]) -> tuple[bytes, int]:
subscription = capture.get("subscription")
notify = capture.get("notify")
if not isinstance(subscription, dict) or not isinstance(notify, dict):
raise ValueError("subscription and notify are required to bind the share")
extra1 = hex_field(subscription.get("extranonce1"))
extra_size = subscription.get("extranonce2_size")
if (len(extra1) > 32 or type(extra_size) is not int
or not 0 <= extra_size <= 32):
raise ValueError("invalid subscription extranonce")
extra2 = hex_field(params[2], extra_size)
job = notify.get("params")
if (notify.get("method") != "mining.notify" or not isinstance(job, list)
or len(job) != 9 or job[0] != params[1]
or not isinstance(job[4], list) or len(job[4]) > 64
or type(job[8]) is not bool):
raise ValueError("submission is not bound to a matching notify")
prevhash = hex_field(job[1], 32)
coinbase = hex_field(job[2]) + extra1 + extra2 + hex_field(job[3])
if len(coinbase) > 65_536:
raise ValueError("coinbase exceeds supported size")
root = hashlib.sha256(hashlib.sha256(coinbase).digest()).digest()
for branch in job[4]:
pair = root + hex_field(branch, 32)
root = hashlib.sha256(hashlib.sha256(pair).digest()).digest()
base_version = u32_hex(job[5])
nbits = u32_hex(job[6])
u32_hex(job[7])
submitted_ntime = u32_hex(params[3])
submitted_nonce = u32_hex(params[4])
if len(params) == 6:
configure = capture.get("configure_response")
result = configure.get("result") if isinstance(configure, dict) else None
if not isinstance(result, dict) or result.get("version-rolling") is not True:
raise ValueError("version rolling was not negotiated")
mask_text = result.get("version-rolling.mask")
mask_event = capture.get("version_mask_event")
if mask_event is not None:
fields = mask_event.get("params") if isinstance(mask_event, dict) else None
if (not isinstance(mask_event, dict)
or mask_event.get("method") != "mining.set_version_mask"
or not isinstance(fields, list) or len(fields) != 1):
raise ValueError("invalid effective version mask event")
mask_text = fields[0]
mask = u32_hex(mask_text)
bits = u32_hex(params[5])
if bits & ~mask:
raise ValueError("submitted version bits exceed negotiated mask")
version = (base_version & ~mask) | bits
else:
if capture.get("configure_response") is not None or capture.get("version_mask_event") is not None:
raise ValueError("negotiated version mask requires submission bits")
version = base_version
header = bytearray()
header.extend(version.to_bytes(4, "little"))
for offset in range(0, 32, 4):
header.extend(prevhash[offset:offset + 4][::-1])
header.extend(root)
header.extend(submitted_ntime.to_bytes(4, "little"))
header.extend(nbits.to_bytes(4, "little"))
header.extend(submitted_nonce.to_bytes(4, "little"))
return bytes(header), submitted_nonce
def proof_digest(header: bytes, algorithm: str) -> bytes:
if algorithm == "Sha256":
return hashlib.sha256(hashlib.sha256(header).digest()).digest()
if algorithm == "Scrypt":
return hashlib.scrypt(header, salt=header, n=1024, r=1, p=1, dklen=32)
raise ValueError(f"{algorithm} share verifier is not implemented")
def verify_accept(capture: dict, algorithm: str) -> None:
if algorithm not in ("Sha256", "Scrypt"):
raise ValueError(f"{algorithm} share verifier is not implemented")
if capture.get("nonce_source") != "real_silicon_capture":
raise ValueError("share was not sourced from captured silicon")
header_hex = capture.get("header80")
target_hex = capture.get("pool_target_le")
nonce = capture.get("nonce")
if (not isinstance(header_hex, str) or len(header_hex) != 160
or not isinstance(target_hex, str) or not DIGEST.fullmatch(target_hex)
or type(nonce) is not int or not 0 <= nonce <= 0xffffffff):
raise ValueError("invalid accepted share work")
try:
header = bytearray.fromhex(header_hex)
except ValueError as error:
raise ValueError("invalid accepted share header") from error
if int.from_bytes(header[76:80], "little") != nonce:
raise ValueError("nonce differs from submitted header")
submission = capture.get("submission")
response = capture.get("pool_response")
params = submission.get("params") if isinstance(submission, dict) else None
if (not isinstance(params, list) or len(params) not in (5, 6)
or not all(isinstance(value, str) for value in params)
or not params[0] or not params[1]
or params[3].lower() != f"{int.from_bytes(header[68:72], 'little'):08x}"
or params[4].lower() != f"{nonce:08x}"
or (len(params) == 6 and not re.fullmatch(r"[0-9a-f]{8}", params[5].lower()))
or not isinstance(response, dict)
or submission.get("method") != "mining.submit"
or type(submission.get("id")) is not int
or response.get("id") != submission["id"]
or response.get("result") is not True
or response.get("error") is not None):
raise ValueError("no matching pool ACCEPT for submitted nonce")
target = int.from_bytes(bytes.fromhex(target_hex), "little")
if not 0 < target <= DIFF_ONE_TARGET:
raise ValueError("pool target is below difficulty one")
difficulty_text = capture.get("difficulty")
if not isinstance(difficulty_text, str) or len(difficulty_text) > 128:
raise ValueError("effective pool difficulty is missing")
try:
difficulty = Decimal(difficulty_text)
except InvalidOperation as error:
raise ValueError("invalid pool difficulty") from error
if not difficulty.is_finite() or difficulty < 1:
raise ValueError("pool difficulty is below one")
if abs(difficulty.as_tuple().exponent) > 64:
raise ValueError("pool difficulty exceeds supported range")
numerator, denominator = difficulty.as_integer_ratio()
expected_target = DIFF_ONE_TARGET * denominator // numerator
if target != expected_target:
raise ValueError("pool target differs from effective difficulty")
rebuilt, submitted_nonce = reconstructed_header(capture, params)
if submitted_nonce != nonce or rebuilt != header:
raise ValueError("submitted Stratum job differs from captured header")
hash_bytes = proof_digest(header, algorithm)
if int.from_bytes(hash_bytes, "little") > target:
raise ValueError("captured nonce misses the pool target")
def verify_cell(row: dict, root: pathlib.Path, run_id: str,
source_commit: str, binary_digest: str,
algorithm: str) -> None:
board, chip = row["board"], row["chip"]
report_path = (root / run_id / board / f"{chip}.json").resolve()
if not report_path.is_relative_to(root.resolve()):
raise ValueError("evidence report is outside evidence directory")
report = load_report(report_path)
identity = {"schema": 1, "run_id": run_id, "source_commit": source_commit,
"os_binary_sha256": binary_digest, "board": board, "chip": chip,
"stock_firmware_modified": False, "algorithm": algorithm}
if any(report.get(key) != value for key, value in identity.items()):
raise ValueError("evidence belongs to a different run, source, binary, or cell")
if report.get("stock_result") != row.get("stock_result"):
raise ValueError("stock result differs from manifest")
if report.get("stock_first_rejection") != row.get("stock_first_rejection"):
raise ValueError("stock rejection differs from manifest")
if report["stock_result"] == "rejects" and not report.get("stock_first_rejection"):
raise ValueError("stock rejection gate is missing")
if report.get("simulator_booted_stock") is not True:
raise ValueError("unmodified stock boot is unverified")
paths = {}
for field, digest_field in (("sdk_image", "sdk_image_sha256"),
("stock_image", "stock_image_sha256"),
("simulator_trace", "simulator_trace_sha256")):
paths[field] = artifact(root, report.get(field))
if report[field]["sha256"] != row.get(digest_field):
raise ValueError(f"{field} digest differs from manifest")
artifact(root, report.get("simulator_model"))
model_digest = report["simulator_model"]["sha256"]
for field in ("control_capture", "chip_capture"):
artifact(root, report.get(field))
trace = load_report(paths["simulator_trace"])
trace_identity = {"run_id": run_id, "source_commit": source_commit,
"board": board, "chip": chip,
"sdk_image_sha256": row["sdk_image_sha256"],
"stock_image_sha256": row["stock_image_sha256"],
"stock_firmware_modified": False,
"stock_result": row["stock_result"],
"algorithm": algorithm,
"simulator_model_sha256": model_digest}
if any(trace.get(key) != value for key, value in trace_identity.items()):
raise ValueError("simulator trace does not bind stock image and current run")
if report["stock_result"] == "rejects":
if trace.get("first_rejection") != row["stock_first_rejection"]:
raise ValueError("stock rejection gate is absent from simulator trace")
else:
stock_capture = load_report(artifact(root, report.get("stock_pool_accept_capture")))
if trace.get("stock_pool_accept_capture_sha256") != report["stock_pool_accept_capture"]["sha256"]:
raise ValueError("stock pool acceptance is not linked to simulator trace")
if (stock_capture.get("board") != board or stock_capture.get("chip") != chip
or stock_capture.get("firmware") != "stock"):
raise ValueError("stock pool acceptance belongs to another firmware or cell")
verify_accept(stock_capture, algorithm)
capture = load_report(artifact(root, report.get("alkane_share_accept_capture")))
if report["alkane_share_accept_capture"]["path"] != row.get("alkane_share_accept_capture"):
raise ValueError("accepted share artifact differs from manifest")
if trace.get("alkane_share_accept_capture_sha256") != report["alkane_share_accept_capture"]["sha256"]:
raise ValueError("Alkane pool acceptance is not linked to simulator trace")
if (capture.get("board") != board or capture.get("chip") != chip
or capture.get("firmware") != "alkane"):
raise ValueError("accepted share belongs to another cell")
verify_accept(capture, algorithm)
bench = load_report(artifact(root, report.get("bench_validation")))
if report["bench_validation"]["path"] != row.get("bench_validation"):
raise ValueError("bench artifact differs from manifest")
if (bench.get("board") != board or bench.get("chip") != chip
or bench.get("validated") is not True
or bench.get("simulator_model_sha256") != model_digest
or any(bench.get(f"{field}_sha256") != report[field]["sha256"]
for field in ("control_capture", "chip_capture"))
or any(not isinstance(bench.get(f"{name}_device_id"), str)
or not bench[f"{name}_device_id"]
for name in ("control", "chip"))):
raise ValueError("bench validation is missing or belongs to another cell")
def binary_targets(path: pathlib.Path) -> dict[tuple[str, str], str]:
output = subprocess.run(
[str(path), "--list-targets"],
check=True, capture_output=True, text=True, timeout=15,
).stdout.splitlines()
if not output or output[0] != "control_board,hash_chip,algorithm":
raise ValueError("Alkane OS binary has no target matrix header")
targets = {}
for line in output[1:]:
parts = line.split(",")
if len(parts) != 3 or not all(parts):
raise ValueError(f"invalid binary target row: {line!r}")
cell = (parts[0], parts[1])
if cell in targets:
raise ValueError("duplicate binary target")
targets[cell] = parts[2]
return targets
def validate(manifest: dict, built_targets: dict[tuple[str, str], str],
evidence_root: pathlib.Path = pathlib.Path("ci/evidence"),
run_id: str = "", source_commit: str = "",
binary_digest: str = "") -> list[str]:
errors = []
if manifest.get("schema") != 1:
errors.append("unsupported conformance manifest schema")
boards = set(manifest["boards"])
chips = set(manifest["chips"])
vendors = manifest["vendors"]
if len(boards) != len(manifest["boards"]) or len(chips) != len(manifest["chips"]):
errors.append("duplicate board or chip in manifest")
for name, vendor in {**REQUIRED_BOARDS, **REQUIRED_CHIPS}.items():
if name not in boards | chips or vendors.get(name) != vendor:
errors.append(f"required target identity changed: {name}")
if set(vendors) != boards | chips:
errors.append("vendor map differs from board and chip lists")
expected = {(board, chip) for board in boards for chip in chips
if vendors.get(board) == vendors.get(chip)}
rows = manifest["targets"]
actual = {(row["board"], row["chip"]) for row in rows}
if len(expected) < 64 or actual != expected or len(rows) != len(expected):
errors.append(f"manifest matrix mismatch: {len(actual)} present, {len(expected)} expected")
if set(built_targets) != expected:
errors.append(f"binary matrix mismatch: {len(built_targets)} built, {len(expected)} expected")
for (board, chip), algorithm in built_targets.items():
if chip in REQUIRED_ALGORITHMS and algorithm != REQUIRED_ALGORITHMS[chip]:
errors.append(f"binary algorithm mismatch: {board} x {chip}")
if (not run_id.isdecimal() or not COMMIT.fullmatch(source_commit)
or not DIGEST.fullmatch(binary_digest)):
errors.append("current run, source commit, or binary digest is unbound")
required = ("sdk_image_sha256", "stock_image_sha256", "stock_result",
"alkane_share_accept_capture", "simulator_trace_sha256",
"bench_validation")
for row in rows:
cell = f"{row['board']} x {row['chip']}"
missing = [field for field in required if not row.get(field)]
if row.get("stock_result") not in (None, "mines", "rejects"):
missing.append("stock_result invalid")
if row.get("stock_result") == "rejects" and not row.get("stock_first_rejection"):
missing.append("stock_first_rejection")
for field in ("sdk_image_sha256", "stock_image_sha256", "simulator_trace_sha256"):
value = row.get(field)
if value and (not isinstance(value, str) or not DIGEST.fullmatch(value)):
missing.append(f"{field} invalid")
if missing:
errors.append(f"{cell}: {', '.join(missing)}")
elif run_id.isdecimal() and COMMIT.fullmatch(source_commit) and DIGEST.fullmatch(binary_digest):
try:
verify_cell(row, evidence_root, run_id, source_commit, binary_digest,
built_targets.get((row["board"], row["chip"]), "unknown"))
except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as error:
errors.append(f"{cell}: evidence invalid: {error}")
return errors
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=pathlib.Path,
default=pathlib.Path("ci/conformance.json"))
parser.add_argument("--binary", type=pathlib.Path,
default=pathlib.Path(os.environ.get("CARGO_TARGET_DIR", "target"))
/ "release/alkane-os")
parser.add_argument("--evidence-dir", type=pathlib.Path,
default=pathlib.Path("ci/evidence"))
parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", ""))
args = parser.parse_args()
manifest = json.loads(args.manifest.read_text())
try:
built_targets = binary_targets(args.binary.resolve())
except (OSError, subprocess.SubprocessError, ValueError) as error:
sys.exit(f"cannot verify Alkane OS binary target matrix: {error}")
source_commit = subprocess.run(["git", "rev-parse", "HEAD"], check=True,
capture_output=True, text=True).stdout.strip()
errors = validate(manifest, built_targets, args.evidence_dir, args.run_id,
source_commit, sha256_file(args.binary.resolve()))
if errors:
print(f"Conformance pending: {len(errors)} issues across {len(manifest['targets'])} targets")
for error in errors[:8]:
print(f" {error}")
sys.exit(1)
print(f"Conformance complete: {len(manifest['targets'])} targets")
if __name__ == "__main__":
main()