402 lines
20 KiB
Python
402 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: MIT
|
|
"""Require the binary target matrix and traceable per-cell release evidence."""
|
|
|
|
import argparse
|
|
from decimal import Decimal, InvalidOperation
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
REQUIRED_BOARDS = {
|
|
"am335x-bb": "bitmain",
|
|
"zynq-7010": "bitmain",
|
|
"cv1835": "bitmain",
|
|
"a113d": "bitmain",
|
|
"allwinner-h3": "whatsminer",
|
|
"allwinner-h6": "whatsminer",
|
|
"allwinner-h616": "whatsminer",
|
|
}
|
|
REQUIRED_CHIPS = {
|
|
**{chip: "bitmain" for chip in (
|
|
"BM1384", "BM1387", "BM1360", "BM1362", "BM1366", "BM1368",
|
|
"BM1370", "BM1396", "BM1397", "BM1398", "BM1489", "BM1491", "BM1746",
|
|
)},
|
|
**{chip: "whatsminer" for chip in (
|
|
"KF1950", "ID-1968", "ID-1980", "ID-1990",
|
|
)},
|
|
}
|
|
REQUIRED_ALGORITHMS = {
|
|
**{chip: "Sha256" for chip in REQUIRED_CHIPS if chip not in
|
|
("BM1489", "BM1491", "BM1746")},
|
|
"BM1489": "Scrypt", "BM1491": "Scrypt", "BM1746": "Equihash",
|
|
}
|
|
DIGEST = re.compile(r"[0-9a-f]{64}\Z")
|
|
COMMIT = re.compile(r"[0-9a-f]{40}\Z")
|
|
DIFF_ONE_TARGET = 0xffff << (8 * (0x1d - 3))
|
|
MAX_REPORT_BYTES = 1_048_576
|
|
|
|
|
|
def sha256_file(path: pathlib.Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for chunk in iter(lambda: source.read(1_048_576), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def load_report(path: pathlib.Path) -> dict:
|
|
if path.stat().st_size > MAX_REPORT_BYTES:
|
|
raise ValueError("evidence report exceeds 1 MiB")
|
|
value = json.loads(path.read_text())
|
|
if not isinstance(value, dict):
|
|
raise ValueError("evidence report is not an object")
|
|
return value
|
|
|
|
|
|
def artifact(root: pathlib.Path, value: object) -> pathlib.Path:
|
|
if not isinstance(value, dict) or not isinstance(value.get("path"), str):
|
|
raise ValueError("artifact reference is missing")
|
|
digest = value.get("sha256")
|
|
if not isinstance(digest, str) or not DIGEST.fullmatch(digest):
|
|
raise ValueError("artifact digest is invalid")
|
|
path = (root / value["path"]).resolve()
|
|
if not path.is_relative_to(root.resolve()) or not path.is_file():
|
|
raise ValueError("artifact is missing or outside evidence directory")
|
|
if sha256_file(path) != digest:
|
|
raise ValueError("artifact digest mismatch")
|
|
return path
|
|
|
|
|
|
def hex_field(value: object, size: int | None = None) -> bytes:
|
|
if (not isinstance(value, str) or len(value) & 1
|
|
or len(value) > 131_072 or not re.fullmatch(r"[0-9a-fA-F]*", value)
|
|
or (size is not None and len(value) != size * 2)):
|
|
raise ValueError("invalid Stratum hex field")
|
|
return bytes.fromhex(value)
|
|
|
|
|
|
def u32_hex(value: object) -> int:
|
|
return int.from_bytes(hex_field(value, 4), "big")
|
|
|
|
|
|
def reconstructed_header(capture: dict, params: list[str]) -> tuple[bytes, int]:
|
|
subscription = capture.get("subscription")
|
|
notify = capture.get("notify")
|
|
if not isinstance(subscription, dict) or not isinstance(notify, dict):
|
|
raise ValueError("subscription and notify are required to bind the share")
|
|
extra1 = hex_field(subscription.get("extranonce1"))
|
|
extra_size = subscription.get("extranonce2_size")
|
|
if (len(extra1) > 32 or type(extra_size) is not int
|
|
or not 0 <= extra_size <= 32):
|
|
raise ValueError("invalid subscription extranonce")
|
|
extra2 = hex_field(params[2], extra_size)
|
|
job = notify.get("params")
|
|
if (notify.get("method") != "mining.notify" or not isinstance(job, list)
|
|
or len(job) != 9 or job[0] != params[1]
|
|
or not isinstance(job[4], list) or len(job[4]) > 64
|
|
or type(job[8]) is not bool):
|
|
raise ValueError("submission is not bound to a matching notify")
|
|
prevhash = hex_field(job[1], 32)
|
|
coinbase = hex_field(job[2]) + extra1 + extra2 + hex_field(job[3])
|
|
if len(coinbase) > 65_536:
|
|
raise ValueError("coinbase exceeds supported size")
|
|
root = hashlib.sha256(hashlib.sha256(coinbase).digest()).digest()
|
|
for branch in job[4]:
|
|
pair = root + hex_field(branch, 32)
|
|
root = hashlib.sha256(hashlib.sha256(pair).digest()).digest()
|
|
base_version = u32_hex(job[5])
|
|
nbits = u32_hex(job[6])
|
|
u32_hex(job[7])
|
|
submitted_ntime = u32_hex(params[3])
|
|
submitted_nonce = u32_hex(params[4])
|
|
if len(params) == 6:
|
|
configure = capture.get("configure_response")
|
|
result = configure.get("result") if isinstance(configure, dict) else None
|
|
if not isinstance(result, dict) or result.get("version-rolling") is not True:
|
|
raise ValueError("version rolling was not negotiated")
|
|
mask_text = result.get("version-rolling.mask")
|
|
mask_event = capture.get("version_mask_event")
|
|
if mask_event is not None:
|
|
fields = mask_event.get("params") if isinstance(mask_event, dict) else None
|
|
if (not isinstance(mask_event, dict)
|
|
or mask_event.get("method") != "mining.set_version_mask"
|
|
or not isinstance(fields, list) or len(fields) != 1):
|
|
raise ValueError("invalid effective version mask event")
|
|
mask_text = fields[0]
|
|
mask = u32_hex(mask_text)
|
|
bits = u32_hex(params[5])
|
|
if bits & ~mask:
|
|
raise ValueError("submitted version bits exceed negotiated mask")
|
|
version = (base_version & ~mask) | bits
|
|
else:
|
|
if capture.get("configure_response") is not None or capture.get("version_mask_event") is not None:
|
|
raise ValueError("negotiated version mask requires submission bits")
|
|
version = base_version
|
|
header = bytearray()
|
|
header.extend(version.to_bytes(4, "little"))
|
|
for offset in range(0, 32, 4):
|
|
header.extend(prevhash[offset:offset + 4][::-1])
|
|
header.extend(root)
|
|
header.extend(submitted_ntime.to_bytes(4, "little"))
|
|
header.extend(nbits.to_bytes(4, "little"))
|
|
header.extend(submitted_nonce.to_bytes(4, "little"))
|
|
return bytes(header), submitted_nonce
|
|
|
|
|
|
def proof_digest(header: bytes, algorithm: str) -> bytes:
|
|
if algorithm == "Sha256":
|
|
return hashlib.sha256(hashlib.sha256(header).digest()).digest()
|
|
if algorithm == "Scrypt":
|
|
return hashlib.scrypt(header, salt=header, n=1024, r=1, p=1, dklen=32)
|
|
raise ValueError(f"{algorithm} share verifier is not implemented")
|
|
|
|
|
|
def verify_accept(capture: dict, algorithm: str) -> None:
|
|
if algorithm not in ("Sha256", "Scrypt"):
|
|
raise ValueError(f"{algorithm} share verifier is not implemented")
|
|
if capture.get("nonce_source") != "real_silicon_capture":
|
|
raise ValueError("share was not sourced from captured silicon")
|
|
header_hex = capture.get("header80")
|
|
target_hex = capture.get("pool_target_le")
|
|
nonce = capture.get("nonce")
|
|
if (not isinstance(header_hex, str) or len(header_hex) != 160
|
|
or not isinstance(target_hex, str) or not DIGEST.fullmatch(target_hex)
|
|
or type(nonce) is not int or not 0 <= nonce <= 0xffffffff):
|
|
raise ValueError("invalid accepted share work")
|
|
try:
|
|
header = bytearray.fromhex(header_hex)
|
|
except ValueError as error:
|
|
raise ValueError("invalid accepted share header") from error
|
|
if int.from_bytes(header[76:80], "little") != nonce:
|
|
raise ValueError("nonce differs from submitted header")
|
|
submission = capture.get("submission")
|
|
response = capture.get("pool_response")
|
|
params = submission.get("params") if isinstance(submission, dict) else None
|
|
if (not isinstance(params, list) or len(params) not in (5, 6)
|
|
or not all(isinstance(value, str) for value in params)
|
|
or not params[0] or not params[1]
|
|
or params[3].lower() != f"{int.from_bytes(header[68:72], 'little'):08x}"
|
|
or params[4].lower() != f"{nonce:08x}"
|
|
or (len(params) == 6 and not re.fullmatch(r"[0-9a-f]{8}", params[5].lower()))
|
|
or not isinstance(response, dict)
|
|
or submission.get("method") != "mining.submit"
|
|
or type(submission.get("id")) is not int
|
|
or response.get("id") != submission["id"]
|
|
or response.get("result") is not True
|
|
or response.get("error") is not None):
|
|
raise ValueError("no matching pool ACCEPT for submitted nonce")
|
|
target = int.from_bytes(bytes.fromhex(target_hex), "little")
|
|
if not 0 < target <= DIFF_ONE_TARGET:
|
|
raise ValueError("pool target is below difficulty one")
|
|
difficulty_text = capture.get("difficulty")
|
|
if not isinstance(difficulty_text, str) or len(difficulty_text) > 128:
|
|
raise ValueError("effective pool difficulty is missing")
|
|
try:
|
|
difficulty = Decimal(difficulty_text)
|
|
except InvalidOperation as error:
|
|
raise ValueError("invalid pool difficulty") from error
|
|
if not difficulty.is_finite() or difficulty < 1:
|
|
raise ValueError("pool difficulty is below one")
|
|
if abs(difficulty.as_tuple().exponent) > 64:
|
|
raise ValueError("pool difficulty exceeds supported range")
|
|
numerator, denominator = difficulty.as_integer_ratio()
|
|
expected_target = DIFF_ONE_TARGET * denominator // numerator
|
|
if target != expected_target:
|
|
raise ValueError("pool target differs from effective difficulty")
|
|
rebuilt, submitted_nonce = reconstructed_header(capture, params)
|
|
if submitted_nonce != nonce or rebuilt != header:
|
|
raise ValueError("submitted Stratum job differs from captured header")
|
|
hash_bytes = proof_digest(header, algorithm)
|
|
if int.from_bytes(hash_bytes, "little") > target:
|
|
raise ValueError("captured nonce misses the pool target")
|
|
|
|
|
|
def verify_cell(row: dict, root: pathlib.Path, run_id: str,
|
|
source_commit: str, binary_digest: str,
|
|
algorithm: str) -> None:
|
|
board, chip = row["board"], row["chip"]
|
|
report_path = (root / run_id / board / f"{chip}.json").resolve()
|
|
if not report_path.is_relative_to(root.resolve()):
|
|
raise ValueError("evidence report is outside evidence directory")
|
|
report = load_report(report_path)
|
|
identity = {"schema": 1, "run_id": run_id, "source_commit": source_commit,
|
|
"os_binary_sha256": binary_digest, "board": board, "chip": chip,
|
|
"stock_firmware_modified": False, "algorithm": algorithm}
|
|
if any(report.get(key) != value for key, value in identity.items()):
|
|
raise ValueError("evidence belongs to a different run, source, binary, or cell")
|
|
if report.get("stock_result") != row.get("stock_result"):
|
|
raise ValueError("stock result differs from manifest")
|
|
if report.get("stock_first_rejection") != row.get("stock_first_rejection"):
|
|
raise ValueError("stock rejection differs from manifest")
|
|
if report["stock_result"] == "rejects" and not report.get("stock_first_rejection"):
|
|
raise ValueError("stock rejection gate is missing")
|
|
if report.get("simulator_booted_stock") is not True:
|
|
raise ValueError("unmodified stock boot is unverified")
|
|
paths = {}
|
|
for field, digest_field in (("sdk_image", "sdk_image_sha256"),
|
|
("stock_image", "stock_image_sha256"),
|
|
("simulator_trace", "simulator_trace_sha256")):
|
|
paths[field] = artifact(root, report.get(field))
|
|
if report[field]["sha256"] != row.get(digest_field):
|
|
raise ValueError(f"{field} digest differs from manifest")
|
|
artifact(root, report.get("simulator_model"))
|
|
model_digest = report["simulator_model"]["sha256"]
|
|
for field in ("control_capture", "chip_capture"):
|
|
artifact(root, report.get(field))
|
|
trace = load_report(paths["simulator_trace"])
|
|
trace_identity = {"run_id": run_id, "source_commit": source_commit,
|
|
"board": board, "chip": chip,
|
|
"sdk_image_sha256": row["sdk_image_sha256"],
|
|
"stock_image_sha256": row["stock_image_sha256"],
|
|
"stock_firmware_modified": False,
|
|
"stock_result": row["stock_result"],
|
|
"algorithm": algorithm,
|
|
"simulator_model_sha256": model_digest}
|
|
if any(trace.get(key) != value for key, value in trace_identity.items()):
|
|
raise ValueError("simulator trace does not bind stock image and current run")
|
|
if report["stock_result"] == "rejects":
|
|
if trace.get("first_rejection") != row["stock_first_rejection"]:
|
|
raise ValueError("stock rejection gate is absent from simulator trace")
|
|
else:
|
|
stock_capture = load_report(artifact(root, report.get("stock_pool_accept_capture")))
|
|
if trace.get("stock_pool_accept_capture_sha256") != report["stock_pool_accept_capture"]["sha256"]:
|
|
raise ValueError("stock pool acceptance is not linked to simulator trace")
|
|
if (stock_capture.get("board") != board or stock_capture.get("chip") != chip
|
|
or stock_capture.get("firmware") != "stock"):
|
|
raise ValueError("stock pool acceptance belongs to another firmware or cell")
|
|
verify_accept(stock_capture, algorithm)
|
|
capture = load_report(artifact(root, report.get("alkane_share_accept_capture")))
|
|
if report["alkane_share_accept_capture"]["path"] != row.get("alkane_share_accept_capture"):
|
|
raise ValueError("accepted share artifact differs from manifest")
|
|
if trace.get("alkane_share_accept_capture_sha256") != report["alkane_share_accept_capture"]["sha256"]:
|
|
raise ValueError("Alkane pool acceptance is not linked to simulator trace")
|
|
if (capture.get("board") != board or capture.get("chip") != chip
|
|
or capture.get("firmware") != "alkane"):
|
|
raise ValueError("accepted share belongs to another cell")
|
|
verify_accept(capture, algorithm)
|
|
bench = load_report(artifact(root, report.get("bench_validation")))
|
|
if report["bench_validation"]["path"] != row.get("bench_validation"):
|
|
raise ValueError("bench artifact differs from manifest")
|
|
if (bench.get("board") != board or bench.get("chip") != chip
|
|
or bench.get("validated") is not True
|
|
or bench.get("simulator_model_sha256") != model_digest
|
|
or any(bench.get(f"{field}_sha256") != report[field]["sha256"]
|
|
for field in ("control_capture", "chip_capture"))
|
|
or any(not isinstance(bench.get(f"{name}_device_id"), str)
|
|
or not bench[f"{name}_device_id"]
|
|
for name in ("control", "chip"))):
|
|
raise ValueError("bench validation is missing or belongs to another cell")
|
|
|
|
|
|
def binary_targets(path: pathlib.Path) -> dict[tuple[str, str], str]:
|
|
output = subprocess.run(
|
|
[str(path), "--list-targets"],
|
|
check=True, capture_output=True, text=True, timeout=15,
|
|
).stdout.splitlines()
|
|
if not output or output[0] != "control_board,hash_chip,algorithm":
|
|
raise ValueError("Alkane OS binary has no target matrix header")
|
|
targets = {}
|
|
for line in output[1:]:
|
|
parts = line.split(",")
|
|
if len(parts) != 3 or not all(parts):
|
|
raise ValueError(f"invalid binary target row: {line!r}")
|
|
cell = (parts[0], parts[1])
|
|
if cell in targets:
|
|
raise ValueError("duplicate binary target")
|
|
targets[cell] = parts[2]
|
|
return targets
|
|
|
|
|
|
def validate(manifest: dict, built_targets: dict[tuple[str, str], str],
|
|
evidence_root: pathlib.Path = pathlib.Path("ci/evidence"),
|
|
run_id: str = "", source_commit: str = "",
|
|
binary_digest: str = "") -> list[str]:
|
|
errors = []
|
|
if manifest.get("schema") != 1:
|
|
errors.append("unsupported conformance manifest schema")
|
|
boards = set(manifest["boards"])
|
|
chips = set(manifest["chips"])
|
|
vendors = manifest["vendors"]
|
|
if len(boards) != len(manifest["boards"]) or len(chips) != len(manifest["chips"]):
|
|
errors.append("duplicate board or chip in manifest")
|
|
for name, vendor in {**REQUIRED_BOARDS, **REQUIRED_CHIPS}.items():
|
|
if name not in boards | chips or vendors.get(name) != vendor:
|
|
errors.append(f"required target identity changed: {name}")
|
|
if set(vendors) != boards | chips:
|
|
errors.append("vendor map differs from board and chip lists")
|
|
expected = {(board, chip) for board in boards for chip in chips
|
|
if vendors.get(board) == vendors.get(chip)}
|
|
rows = manifest["targets"]
|
|
actual = {(row["board"], row["chip"]) for row in rows}
|
|
if len(expected) < 64 or actual != expected or len(rows) != len(expected):
|
|
errors.append(f"manifest matrix mismatch: {len(actual)} present, {len(expected)} expected")
|
|
if set(built_targets) != expected:
|
|
errors.append(f"binary matrix mismatch: {len(built_targets)} built, {len(expected)} expected")
|
|
for (board, chip), algorithm in built_targets.items():
|
|
if chip in REQUIRED_ALGORITHMS and algorithm != REQUIRED_ALGORITHMS[chip]:
|
|
errors.append(f"binary algorithm mismatch: {board} x {chip}")
|
|
if (not run_id.isdecimal() or not COMMIT.fullmatch(source_commit)
|
|
or not DIGEST.fullmatch(binary_digest)):
|
|
errors.append("current run, source commit, or binary digest is unbound")
|
|
required = ("sdk_image_sha256", "stock_image_sha256", "stock_result",
|
|
"alkane_share_accept_capture", "simulator_trace_sha256",
|
|
"bench_validation")
|
|
for row in rows:
|
|
cell = f"{row['board']} x {row['chip']}"
|
|
missing = [field for field in required if not row.get(field)]
|
|
if row.get("stock_result") not in (None, "mines", "rejects"):
|
|
missing.append("stock_result invalid")
|
|
if row.get("stock_result") == "rejects" and not row.get("stock_first_rejection"):
|
|
missing.append("stock_first_rejection")
|
|
for field in ("sdk_image_sha256", "stock_image_sha256", "simulator_trace_sha256"):
|
|
value = row.get(field)
|
|
if value and (not isinstance(value, str) or not DIGEST.fullmatch(value)):
|
|
missing.append(f"{field} invalid")
|
|
if missing:
|
|
errors.append(f"{cell}: {', '.join(missing)}")
|
|
elif run_id.isdecimal() and COMMIT.fullmatch(source_commit) and DIGEST.fullmatch(binary_digest):
|
|
try:
|
|
verify_cell(row, evidence_root, run_id, source_commit, binary_digest,
|
|
built_targets.get((row["board"], row["chip"]), "unknown"))
|
|
except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as error:
|
|
errors.append(f"{cell}: evidence invalid: {error}")
|
|
return errors
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--manifest", type=pathlib.Path,
|
|
default=pathlib.Path("ci/conformance.json"))
|
|
parser.add_argument("--binary", type=pathlib.Path,
|
|
default=pathlib.Path(os.environ.get("CARGO_TARGET_DIR", "target"))
|
|
/ "release/alkane-os")
|
|
parser.add_argument("--evidence-dir", type=pathlib.Path,
|
|
default=pathlib.Path("ci/evidence"))
|
|
parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", ""))
|
|
args = parser.parse_args()
|
|
manifest = json.loads(args.manifest.read_text())
|
|
try:
|
|
built_targets = binary_targets(args.binary.resolve())
|
|
except (OSError, subprocess.SubprocessError, ValueError) as error:
|
|
sys.exit(f"cannot verify Alkane OS binary target matrix: {error}")
|
|
source_commit = subprocess.run(["git", "rev-parse", "HEAD"], check=True,
|
|
capture_output=True, text=True).stdout.strip()
|
|
errors = validate(manifest, built_targets, args.evidence_dir, args.run_id,
|
|
source_commit, sha256_file(args.binary.resolve()))
|
|
if errors:
|
|
print(f"Conformance pending: {len(errors)} issues across {len(manifest['targets'])} targets")
|
|
for error in errors[:8]:
|
|
print(f" {error}")
|
|
sys.exit(1)
|
|
print(f"Conformance complete: {len(manifest['targets'])} targets")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|