Files
alkane-os/ci/afw_manifest.py
T

488 lines
20 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Generate an .afw manifest and stage its payload members.
The output is a full OS install bundle for exactly one board family: the
manifest names every image, its size and SHA-256, the partition plan, the
bootloader policy, the network recovery path and a liveness marker. Payload
members are staged under <staging>/payload/ in manifest order.
This script does not create the tar. ci/make-afw.sh packs the staged members
in the fixed order manifest.json, optional manifest.json.sig, then payload.
ci/check-afw.py verifies the finished bundle before it is published.
"""
import argparse
import datetime
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import afw_boards
SCHEMA = 2
P1_LOAD = 0x100200000
GOLDEN_LBA_BYTES = 0x300000 * 512
REVISION = 40
AM335X_MAP = {
"mtd0": (0x0000000, 0x20000, True, True, 0, None),
"mtd1": (0x0020000, 0x20000, True, True, 0, None),
"mtd2": (0x0040000, 0x20000, True, True, 0, None),
"mtd3": (0x0060000, 0x20000, True, True, 0, None),
"mtd4": (0x0080000, 0x1C0000, True, True, 0, None),
"mtd5": (0x0240000, 0x20000, True, True, 0, None),
"mtd6": (0x0260000, 0x20000, False, True, 2, "dtb"),
"mtd7": (0x0280000, 0x500000, False, True, 3, "kernel"),
"mtd8": (0x0800000, 0x1400000, False, True, 1, "rootfs"),
"mtd9": (0x1C00000, 0x200000, True, False, 0, None),
"mtd10": (0x1E00000, 0x200000, True, False, 0, None),
"mtd11": (0x2000000, 0x6000000, True, False, 0, None),
}
AM335X_HOLE = (0x780000, 0x80000)
AM335X_CHIP = 0x8000000
AM335X_TARGETS = {"kernel": "mtd7", "dtb": "mtd6", "rootfs": "mtd8"}
AM335X_FORMATS = {"kernel": "uimage", "dtb": "fdt", "rootfs": "uimage"}
AM335X_CEILINGS = {"kernel": 0x500000, "dtb": 0x20000, "rootfs": 0x1400000}
def die(message):
print(f"afw_manifest: error: {message}", file=sys.stderr)
raise SystemExit(1)
def sha256_bytes(data):
return hashlib.sha256(data).hexdigest()
def sha256_file(path):
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def git_revision(path):
try:
return subprocess.check_output(
["git", "-C", str(path), "rev-parse", "HEAD"],
text=True,
stderr=subprocess.DEVNULL,
).strip()
except (OSError, subprocess.CalledProcessError):
return None
def check_revision(name, value):
if value is None or len(value) != REVISION:
die(f"{name} must be a 40-character git revision, got {value!r}")
if any(character not in "0123456789abcdef" for character in value):
die(f"{name} must be lowercase hexadecimal")
return value
def build_date(value):
if value is None:
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
try:
moment = datetime.datetime.fromtimestamp(int(value), datetime.timezone.utc)
except (ValueError, OSError):
die("SOURCE_DATE_EPOCH must be an integer Unix timestamp")
return moment.strftime("%Y-%m-%dT%H:%M:%SZ")
def magic_ok(fmt, path):
with path.open("rb") as source:
data = source.read() if fmt == "fit" else source.read(4096)
if fmt == "fit":
if not data.startswith(b"\xd0\x0d\xfe\xed"):
return "does not start with the FIT/FDT magic d00dfeed"
if b"config@cv1835_miner_emmc" not in data:
return "FIT is missing config@cv1835_miner_emmc"
if b"signature@" in data:
return "FIT carries signature@ nodes, which this board cannot boot"
elif fmt == "fdt":
if not data.startswith(b"\xd0\x0d\xfe\xed"):
return "does not start with the FDT magic d00dfeed"
elif fmt == "arm64-image":
if len(data) < 64 or data[0x38:0x3C] != b"ARMd":
return "does not carry the arm64 Image ARMd header magic"
elif fmt == "cpio-gzip":
if not data.startswith(b"\x1f\x8b"):
return "is not gzip-compressed"
elif fmt == "uimage":
if not data.startswith(b"\x27\x05\x19\x56"):
return "is not a legacy uImage"
return None
def pad_to(source, destination, exact_bytes):
size = source.stat().st_size
if size > exact_bytes:
die(f"{source.name} is {size} bytes, larger than its {exact_bytes}-byte partition")
destination.parent.mkdir(parents=True, exist_ok=True)
with source.open("rb") as reader, destination.open("wb") as writer:
shutil.copyfileobj(reader, writer)
if size < exact_bytes:
writer.write(b"\0" * (exact_bytes - size))
if destination.stat().st_size != exact_bytes:
die(f"{destination.name} did not reach the exact {exact_bytes}-byte size")
def load_plan(path, profile, board):
plan = json.loads(Path(path).read_text())
if plan.get("board") != board:
die(f"plan board {plan.get('board')!r} does not match --board {board!r}")
if plan.get("arch") != profile["arch"]:
die(f"plan arch {plan.get('arch')!r} does not match {profile['arch']!r}")
policy = plan.get("bootloader_policy", profile["bootloader_policy"])
if policy != profile["bootloader_policy"]:
die(f"plan bootloader_policy {policy!r} does not match {profile['bootloader_policy']!r}")
partitions = plan.get("partitions")
images = plan.get("images")
if not isinstance(partitions, list) or not isinstance(images, list):
die("plan must contain partitions and images lists")
return partitions, images
def signing_key(path):
raw = Path(path).read_bytes()
if len(raw) == 64:
try:
raw = bytes.fromhex(raw.decode("ascii").strip())
except (ValueError, UnicodeDecodeError):
die("signing key file must be 32 raw bytes or 64 hexadecimal characters")
if len(raw) != 32:
die("signing key file must be 32 raw bytes or 64 hexadecimal characters")
try:
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
except ImportError:
die("Ed25519 signing needs the Python cryptography package")
private = Ed25519PrivateKey.from_private_bytes(raw)
public = private.public_key().public_bytes(
serialization.Encoding.Raw, serialization.PublicFormat.Raw
)
return private, public.hex()
def validate_am335x(partitions, images):
"""Reject the AM335x brick paths before a bundle is packed."""
installs = [image for image in images if image["install"]]
if len(installs) != 3:
die("am335x must install exactly kernel, dtb and rootfs")
if {image["name"] for image in installs} != set(AM335X_TARGETS):
die("am335x install images must be named kernel, dtb and rootfs")
for image in images:
lowered = f"{image['name']} {image['file']}".lower()
if "spl" in lowered or "u-boot" in lowered or "uboot" in lowered:
die(f"am335x bundle carries an spl/u-boot member: {image['file']!r}")
for image in installs:
name = image["name"]
if image["format"] != AM335X_FORMATS[name]:
die(f"am335x {name} format must be {AM335X_FORMATS[name]}")
if image["partition"] != AM335X_TARGETS[name]:
die(f"am335x {name} must target {AM335X_TARGETS[name]}")
if image["size"] > AM335X_CEILINGS[name]:
die(
f"am335x {name} is {image['size']} bytes, over the "
f"0x{AM335X_CEILINGS[name]:x}-byte ceiling"
)
seen = {}
for partition in partitions:
name = partition["name"]
if name not in AM335X_MAP:
die(f"unknown am335x partition {name!r}")
seen[name] = partition
offset, size, secure, boot_critical, write_order, image_name = AM335X_MAP[name]
if int(partition["offset"]) != offset:
die(f"am335x {name} offset must be 0x{offset:x}")
if int(partition["size"]) != size:
die(f"am335x {name} size must be 0x{size:x}")
if bool(partition["secure"]) is not secure:
die(f"am335x {name} secure flag is wrong")
if bool(partition["boot_critical"]) is not boot_critical:
die(f"am335x {name} boot_critical flag is wrong")
if int(partition.get("write_order", 0)) != write_order:
die(f"am335x {name} write_order must be {write_order}")
if partition.get("image") != image_name:
die(f"am335x {name} image must be {image_name!r}")
start = int(partition["offset"])
end = start + int(partition["size"])
hole_start, hole_size = AM335X_HOLE
if start < hole_start + hole_size and end > hole_start:
die(f"am335x {name} overlaps the 0x780000-0x800000 hole")
if name in {f"mtd{index}" for index in range(0, 6)} and write_order != 0:
die(f"am335x {name} is preserved and must not be written")
if set(seen) != set(AM335X_MAP):
die("am335x manifest partition map is incomplete")
if int(seen["mtd7"]["offset"]) + int(seen["mtd7"]["size"]) != AM335X_HOLE[0]:
die("am335x mtd7 must end at the 0x780000 hole")
if int(seen["mtd8"]["offset"]) != AM335X_HOLE[0] + AM335X_HOLE[1]:
die("am335x mtd8 must start at 0x800000 after the hole")
total = sum(int(part["size"]) for part in partitions)
if total != AM335X_CHIP - AM335X_HOLE[1]:
die("am335x partition sizes must sum to 0x7F80000")
def build(args):
board = args.board
if not afw_boards.known_board(board):
die(f"unknown board family {board!r}")
profile = afw_boards.board(board)
if not profile["buildable"]:
die(f"board {board!r} has no buildable .afw profile in this release")
input_dir = Path(args.input)
if not input_dir.is_dir():
die(f"input directory does not exist: {input_dir}")
plan_document = None
if args.plan:
plan_document = json.loads(Path(args.plan).read_text())
partitions, image_specs = load_plan(args.plan, profile, board)
else:
partitions = profile["partitions"]
image_specs = profile["images"]
staging = Path(args.staging) if args.staging else Path(tempfile.mkdtemp(prefix="afw-manifest."))
payload_dir = staging / "payload"
payload_dir.mkdir(parents=True, exist_ok=True)
images = []
for spec in image_specs:
source_name = spec.get("source", spec.get("file"))
if not source_name:
die(f"image {spec.get('name')!r} has no source file")
source = input_dir / source_name
if not source.is_file():
die(f"required image is missing: {source}")
name = spec["name"]
fmt = spec.get("format", "raw")
if fmt not in afw_boards.SAFE_IMAGE_FORMATS:
die(f"image {name!r} has unsupported format {fmt!r}")
role = spec.get("role", "component")
if role not in afw_boards.SAFE_IMAGE_ROLES:
die(f"image {name!r} has unsupported role {role!r}")
target = payload_dir / source.name
exact = spec.get("exact_bytes")
if exact is not None:
pad_to(source, target, int(exact))
else:
shutil.copyfile(source, target)
problem = magic_ok(fmt, target)
if problem:
die(f"image {name!r} ({target.name}) {problem}")
size = target.stat().st_size
images.append(
{
"name": name,
"file": f"payload/{target.name}",
"role": role,
"format": fmt,
"size": size,
"sha256": sha256_file(target),
"partition": spec.get("partition"),
"install": bool(spec.get("install", False)),
}
)
names = {image["name"] for image in images}
for partition in partitions:
image_name = partition.get("image")
if image_name is not None and image_name not in names:
die(f"partition {partition.get('name')!r} references missing image {image_name!r}")
if not partition.get("secure", False):
size = int(partition.get("size", 0))
if size <= 0:
die(f"partition {partition.get('name')!r} has no positive size")
if partition.get("write_order", 0) > 0 and image_name is None:
die(f"installable partition {partition.get('name')!r} has no image")
device = partition.get("device", "")
if not device.startswith("/dev/"):
die(f"partition {partition.get('name')!r} has an invalid device path")
if "boot0" in device and not partition.get("secure", False):
die("boot0 must be declared secure and must never be a write target")
if "boot0" in device and partition.get("write_order", 0) > 0:
die("boot0 must not be in the write order")
for image in images:
partition_name = image.get("partition")
if partition_name is None:
continue
matches = [item for item in partitions if item.get("name") == partition_name]
if len(matches) != 1:
die(f"image {image['name']!r} targets unknown partition {partition_name!r}")
partition = matches[0]
if partition.get("secure", False):
die(f"image {image['name']!r} targets secure partition {partition_name!r}")
if image["size"] > int(partition["size"]):
die(
f"image {image['name']!r} is {image['size']} bytes, larger than "
f"partition {partition_name!r} ({partition['size']} bytes)"
)
exact = image.get("exact_bytes", partition.get("exact_bytes"))
if exact is not None and image["size"] != int(exact):
die(f"image {image['name']!r} must be exactly {exact} bytes")
installs = [image for image in images if image["install"]]
if not installs:
die("bundle has no installable image")
if board == "cv1835":
if len(installs) != 1 or installs[0]["name"] != "p1":
die("cv1835 must install exactly one image, p1")
if installs[0]["size"] != 16 * 1024 * 1024:
die("cv1835 p1 must be the 16 MiB padded FIT")
elif board == "am335x":
validate_am335x(partitions, images)
policy = profile["bootloader_policy"]
if policy == "preserve" and any(image["role"] == "bootloader" for image in images):
die("bootloader_policy is preserve but the bundle carries a bootloader image")
if policy == "include" and not any(image["role"] == "bootloader" for image in images):
die("bootloader_policy is include but the bundle has no bootloader image")
sdk_revision = check_revision(
"sdk_revision", args.sdk_revision or (git_revision(args.sdk_dir) if args.sdk_dir else None)
)
os_revision = check_revision(
"os_revision", args.os_revision or (git_revision(args.os_dir) if args.os_dir else None)
)
signature = {
"type": "unsigned",
"member": None,
"public_key_hex": None,
}
signer = None
if args.signing_key:
signer, public_hex = signing_key(args.signing_key)
signature = {
"type": "ed25519",
"member": "manifest.json.sig",
"public_key_hex": public_hex,
}
if board == "am335x":
boot_switch = {
"method": "preserved-u-boot-go",
"detail": "mtd0-5 (SPL and our non-RSA U-Boot) are preserved and never "
"written. The update writes mtd8 root, mtd6 dtb and mtd7 kernel as "
"legacy ARM uImages, and the existing boot command loads them.",
}
preflight = {
"required_commands": ["flash_erase", "nandwrite", "nanddump", "md5sum", "sync"],
"forbidden_devices": [f"/dev/mtd{index}" for index in range(0, 6)],
"min_free_bytes": 32 * 1024 * 1024,
"source_os": ["alkane", "ouros", "braiins"],
"held_partitions": ["mtd8", "mtd6", "mtd7"],
"preserve_partitions": [f"mtd{index}" for index in range(0, 6)],
}
recovery = {
"network": "afw restore --host <ip> --held-backup <mtd8.img> --confirm FLASH",
"held_partition": "mtd8",
"held_partitions": ["mtd8", "mtd6", "mtd7"],
}
else:
boot_switch = {
"method": "partition-overwrite",
"detail": "Writing /dev/mmcblk0p1 replaces the FIT. The vendor boot0, "
"BL2, BL31 and U-Boot chain is preserved and never written.",
}
preflight = {
"required_commands": ["dd", "md5sum", "sync"],
"forbidden_devices": ["/dev/mmcblk0", "/dev/mmcblk0boot0", "/dev/mmcblk0boot1"],
"min_free_bytes": 32 * 1024 * 1024,
"source_os": ["stock-bitmain", "vnish", "braiins", "ouros", "alkane"],
}
recovery = {
"network": "afw restore --host <ip> --held-backup <file>",
"held_partition": "p1",
"golden_lba_bytes": GOLDEN_LBA_BYTES,
}
manifest = {
"schema": SCHEMA,
"format": "afw",
"board": board,
"arch": profile["arch"],
"version": args.version,
"build_date": build_date(os.environ.get("SOURCE_DATE_EPOCH")),
"sdk_revision": sdk_revision,
"os_revision": os_revision,
"payload_type": "os",
"bootloader_policy": policy,
"signature": signature,
"partitions": [
{
"name": item["name"],
"device": item["device"],
"offset": int(item.get("offset", 0)),
"size": int(item["size"]),
"secure": bool(item.get("secure", False)),
"boot_critical": bool(item.get("boot_critical", False)),
"write_order": int(item.get("write_order", 0)),
"image": item.get("image"),
}
for item in partitions
],
"images": images,
"boot_switch": boot_switch,
"preflight": preflight,
"recovery": recovery,
"marker": (plan_document or {}).get("marker")
or args.marker
or ("Alkane" if board == "am335x" else "ALKANE"),
"kernel_version": (plan_document or {}).get("kernel_version")
or args.kernel_version
or "6.18.48",
}
if args.notes:
manifest["notes"] = args.notes
manifest_path = staging / "manifest.json"
manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode()
manifest_path.write_bytes(manifest_bytes)
if signer is not None:
signature_bytes = signer.sign(manifest_bytes)
(staging / "manifest.json.sig").write_bytes(signature_bytes)
(staging / "manifest.pub").write_text(signature["public_key_hex"] + "\n")
print(str(manifest_path))
return staging, manifest_path
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--board", required=True)
parser.add_argument("--version", required=True)
parser.add_argument("--input", required=True, help="SDK board output directory")
parser.add_argument("--staging", help="staging directory for manifest.json and payload/")
parser.add_argument("--plan", help="plan JSON emitted by alkane-sdk ci/check-plans.py")
parser.add_argument("--sdk-revision")
parser.add_argument("--os-revision")
parser.add_argument("--sdk-dir")
parser.add_argument("--os-dir", default=".")
parser.add_argument("--signing-key", help="raw 32-byte or 64-hex Ed25519 seed file")
parser.add_argument("--kernel-version")
parser.add_argument("--marker")
parser.add_argument("--notes")
args = parser.parse_args()
try:
build(args)
except SystemExit:
raise
except (OSError, ValueError, KeyError) as error:
die(str(error))
if __name__ == "__main__":
main()