diff --git a/qemu/rootfs/sbin/init b/qemu/rootfs/sbin/init index af442a4..80059c7 100755 --- a/qemu/rootfs/sbin/init +++ b/qemu/rootfs/sbin/init @@ -120,6 +120,50 @@ if [ -x /usr/bin/warden-ui ] && [ -c /dev/fb0 ]; then ) & fi +# Control bridge: the UI's debug FIFO, reachable from OUTSIDE the VM. +# +# warden-ui answers nav/page/stats/hit on /tmp/warden-ui.ctl (warden_debug.c), +# but that FIFO lives in here and a scenario drives the VM from the host. On +# real hardware the same channel is reached over SSH (flare-edge +# tools/warden-ctl); this initramfs is busybox-only and has no sshd, so the +# equivalent seam is a second 16550 that run.sh --ctl exposes as a unix socket +# (pci-serial, the same device the RS485 bridge already rides). One command +# per line in, the FIFO's reply out, and a sentinel line so the reader knows +# the reply is complete without a timeout. The vocabulary is identical on both +# sides of that seam, which is what lets one flow script run against the sim +# and against a panel. +# +# run.sh lists the ctl port before any other pci-serial, so it is always the +# first 8250, and it says so with warden.ctl on the command line. The marker, +# not the mere presence of a ttyS0, is what arms the bridge: a VM launched +# with --rs485 alone also has a ttyS0, and that one is the Modbus wire. +if grep -qw warden.ctl /proc/cmdline && [ -c /dev/ttyS0 ]; then + ctl=/dev/ttyS0 + echo "init: control bridge on $ctl" + ( + # Opened ONCE, read-write, on fd 3. Reopening a serial port per line + # can block on carrier detect; one open at bridge start either works + # or fails visibly on the console. The tty stays in its default cooked + # mode: the host discards echoed input, and a whole line arrives per + # read. + exec 3<> "$ctl" + while IFS= read -r cmd <&3; do + [ -n "$cmd" ] || continue + if [ -p /tmp/warden-ui.ctl ]; then + printf '%s\n' "$cmd" > /tmp/warden-ui.ctl + # The UI polls its FIFO every 100 ms and truncates the reply + # file on each command, so a short settle then a read is the + # same protocol warden-ctl uses over SSH. + sleep 0.3 + cat /tmp/warden-ui.dbg 2>/dev/null >&3 + else + echo "bridge: warden-ui control FIFO not present" >&3 + fi + echo "<>" >&3 + done + ) & +fi + if grep -qw warden.shell /proc/cmdline; then echo "warden.shell: interactive shell (exit powers off)" setsid cttyhack sh diff --git a/qemu/run.sh b/qemu/run.sh index 8d52919..d3a8b8e 100755 --- a/qemu/run.sh +++ b/qemu/run.sh @@ -14,6 +14,9 @@ # "device boots believing 2021" incident class # --rs485 SOCK unix socket chardev for the RS485/Modbus bridge # (pci-serial: needs the virt.fragment kernel) +# --ctl SOCK unix socket to warden-ui's debug channel (nav/page/hit), +# bridged by init from a pci-serial port; the rig-side +# twin of tools/warden-ctl over SSH on a real panel # --watchdog add i6300esb watchdog, reset on expiry (fragment kernel) # --qmp SOCK QMP unix socket (screendump, input-send-event, quit) # --display MODE off (default, -nographic) | on (gtk window) | headless @@ -32,7 +35,7 @@ QEMU_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OUT="${OUT:-$QEMU_DIR/out}" KERNEL="" INITRD="$OUT/initramfs.cpio.gz" DISK="" NO_DISK=0 SLOT="_a" -RTC="" RS485="" WATCHDOG=0 QMP="" DISPLAY_MODE="off" SHELL_FLAG=0 ALLOW_APPLY=0 +RTC="" RS485="" CTL="" WATCHDOG=0 QMP="" DISPLAY_MODE="off" SHELL_FLAG=0 ALLOW_APPLY=0 SSH_PORT=2222 HTTP_PORT=8080 API_PORT=28443 EXTRA=() @@ -45,6 +48,7 @@ while [ $# -gt 0 ]; do --slot) SLOT="${2:?}"; shift 2 ;; --rtc) RTC="${2:?}"; shift 2 ;; --rs485) RS485="${2:?}"; shift 2 ;; + --ctl) CTL="${2:?}"; shift 2 ;; --watchdog) WATCHDOG=1; shift ;; --qmp) QMP="${2:?}"; shift 2 ;; --display) DISPLAY_MODE="${2:?}"; shift 2 ;; @@ -102,9 +106,21 @@ if [ -n "$DISK" ] && [ "$NO_DISK" -eq 0 ]; then -device "virtio-blk-device,drive=vd0" ) fi [ "$SHELL_FLAG" -eq 1 ] && APPEND="$APPEND warden.shell" +# Tells init the FIRST 8250 is the control bridge. Without this marker init +# bridges nothing, so a VM launched with --rs485 alone never has its Modbus +# wire mistaken for the debug channel. +[ -n "$CTL" ] && APPEND="$APPEND warden.ctl" [ "$ALLOW_APPLY" -eq 1 ] && APPEND="$APPEND warden.fwapply" [ -n "$RTC" ] && ARGS+=( -rtc "base=$RTC" ) [ "$WATCHDOG" -eq 1 ] && ARGS+=( -device i6300esb -action watchdog=reset ) +# The UI's debug channel, bridged out of the VM: rootfs/sbin/init relays lines +# between this port and warden-ui's control FIFO (see qmp.py --ctl). Same +# pci-serial the RS485 bridge rides. Listed BEFORE rs485 so that, whichever +# combination is requested, this port enumerates as ttyS0 -- the first port +# init's bridge probes. Put rs485 first and the bridge would be talking to the +# Modbus wire. +[ -n "$CTL" ] && ARGS+=( -chardev "socket,id=ctl,path=$CTL,server=on,wait=off" + -device "pci-serial,chardev=ctl" ) [ -n "$RS485" ] && ARGS+=( -chardev "socket,id=rs485,path=$RS485,server=on,wait=off" -device "pci-serial,chardev=rs485" ) [ -n "$QMP" ] && ARGS+=( -qmp "unix:$QMP,server=on,wait=off" ) diff --git a/qemu/tests/qmp.py b/qemu/tests/qmp.py index e52220c..0af8c8a 100755 --- a/qemu/tests/qmp.py +++ b/qemu/tests/qmp.py @@ -20,10 +20,33 @@ coordinates are PANEL PIXELS (0..size-1, default 720) rather than the sleep SECONDS let animations settle or timers fire echo TEXT progress marker in the driver's own output +With `--ctl SOCK` (the unix socket run.sh --ctl creates; ui-drive.sh passes +it) the script can also ask the UI itself, through the debug channel in +warden_debug.c. The same channel is what tools/warden-ctl reaches over SSH on +a real panel, so these verbs mean the same thing on the rig and on hardware: + + nav MENU[/TAB] jump to a page; fails if the page is unknown + page | stats | hit X Y print the reply, judge nothing + ctl WORDS... raw passthrough for anything the channel grows + assert_page MENU/TAB the active page is exactly this + assert_hit X Y CLASS [TEXT...] + a real tap at X,Y lands on an object of CLASS whose + caption contains TEXT: the pre-tap self-check that + reports a MOVED button as such, not as broken + behaviour + +Every step is recorded to /results.jsonl as ok / fail / fatal. An +assertion mismatch is a `fail` and the script continues, so one run reports +every broken expectation. With `--console LOG` the console is checked after +EVERY step for the stage-2 init's "warden-ui EXITED" line, so a crash is a +`fatal` pinned to the step that caused it rather than a stale picture found at +the end. The exit status is non-zero if anything failed. + A step that names an unknown command is a FATAL error rather than a skip: a silently-ignored line in a scenario is a test that proves nothing. """ import json +import re import socket import sys import time @@ -98,12 +121,106 @@ def do_swipe(s, f, x1, y1, x2, y2, size, ms=400, steps=None): send_events(s, f, [btn_ev(False)]) -def drive(s, f, script_path, outdir, size): +class Ctl: + """The UI's debug channel, reached through run.sh --ctl. + + Inside the VM that channel is a FIFO polled by warden-ui (warden_debug.c); + rootfs/sbin/init bridges it to a pci-serial port that run.sh exposes as a + unix socket. One command line in, the reply out, then a sentinel line. On + real hardware flare-edge tools/warden-ctl reaches the identical FIFO over + SSH, so the vocabulary here is the vocabulary there: a flow script that + runs against this rig runs against a panel. + + A missing or silent channel is an infrastructure fault and raises; it is + never reported as an assertion failure, because a rig that cannot ask is + not evidence about the UI either way. + """ + + SENTINEL = "<>" + + def __init__(self, path, timeout=15.0): + self.sock = socket.socket(socket.AF_UNIX) + self.sock.settimeout(timeout) + self.sock.connect(path) + self.buf = b"" + + def send(self, cmd): + self.sock.sendall((cmd + "\n").encode()) + lines = [] + while True: + nl = self.buf.find(b"\n") + if nl < 0: + chunk = self.sock.recv(4096) + if not chunk: + raise RuntimeError("control channel closed") + self.buf += chunk + continue + line = self.buf[:nl].decode("utf-8", "replace").rstrip("\r") + self.buf = self.buf[nl + 1:] + if line == self.SENTINEL: + break + # The tty is in cooked mode, so the command comes back echoed. + if line.strip() == cmd.strip(): + continue + lines.append(line) + return "\n".join(lines).strip() + + +HIT_RE = re.compile(r'^hit \d+,\d+: (\S+)(?: text="(.*)" box=(-?\d+),(-?\d+),(\d+)x(\d+))?$') + + +def parse_hit(reply): + """-> (cls, text, box) with box as 'x1,y1,WxH', or None for 'nothing'; + raises on an unparseable reply.""" + m = HIT_RE.match(reply) + if not m: + raise RuntimeError(f"unparseable hit reply: {reply!r}") + if m.group(1) == "nothing": + return None + box = f"{m.group(3)},{m.group(4)},{m.group(5)}x{m.group(6)}" if m.group(3) else "" + return m.group(1), m.group(2) or "", box + + +def ui_exited(console_path): + """The stage-2 init announces a dead UI on the console; the framebuffer + does not, it just keeps the last frame. Checked after EVERY step so a crash + is pinned to the step that caused it, not discovered at the end.""" + if not console_path: + return None + try: + with open(console_path, "rb") as fh: + data = fh.read() + except OSError: + return None + i = data.find(b"warden-ui EXITED") + if i < 0: + return None + return data[i:i + 80].split(b"\n", 1)[0].decode("utf-8", "replace") + + +def drive(s, f, script_path, outdir, size, ctl_path=None, console_path=None): import os os.makedirs(outdir, exist_ok=True) with open(script_path) as fh: lines = fh.readlines() + ctl = Ctl(ctl_path) if ctl_path else None + results = open(os.path.join(outdir, "results.jsonl"), "w") + counts = {"ok": 0, "fail": 0, "fatal": 0} + + def record(lineno, cmd, status, detail=""): + counts[status] += 1 + results.write(json.dumps({"line": lineno, "cmd": cmd, "status": status, + "detail": detail}) + "\n") + results.flush() + tag = {"ok": "", "fail": "FAIL ", "fatal": "FATAL "}[status] + print(f" [{lineno}] {tag}{cmd}{(' -- ' + detail) if detail else ''}", flush=True) + + def need_ctl(lineno, cmd): + if ctl is None: + sys.exit(f"FATAL: {script_path}:{lineno}: '{cmd}' needs the control " + f"channel; run with --ctl (ui-drive.sh passes it)") + for lineno, raw in enumerate(lines, 1): line = raw.split("#", 1)[0].strip() if not line: @@ -115,27 +232,84 @@ def drive(s, f, script_path, outdir, size): name = args[0] path = os.path.join(os.path.abspath(outdir), f"{name}.ppm") rpc(s, f, {"execute": "screendump", "arguments": {"filename": path}}) - print(f" [{lineno}] shot {name}", flush=True) + record(lineno, line, "ok") elif cmd == "tap": x, y = int(args[0]), int(args[1]) do_tap(s, f, to_axis(x, size), to_axis(y, size)) - print(f" [{lineno}] tap {x},{y}", flush=True) + record(lineno, line, "ok") elif cmd == "swipe": ms = int(args[4]) if len(args) > 4 else 400 do_swipe(s, f, int(args[0]), int(args[1]), int(args[2]), int(args[3]), size, ms) - print(f" [{lineno}] swipe {args[0]},{args[1]} -> {args[2]},{args[3]}", flush=True) + record(lineno, line, "ok") elif cmd == "fling": do_swipe(s, f, int(args[0]), int(args[1]), int(args[2]), int(args[3]), size, ms=120) - print(f" [{lineno}] fling {args[0]},{args[1]} -> {args[2]},{args[3]}", flush=True) + record(lineno, line, "ok") elif cmd == "sleep": time.sleep(float(args[0])) - print(f" [{lineno}] sleep {args[0]}", flush=True) + record(lineno, line, "ok") elif cmd == "echo": print(f" [{lineno}] {' '.join(args)}", flush=True) + elif cmd == "nav": + need_ctl(lineno, cmd) + reply = ctl.send("nav " + " ".join(args)) + record(lineno, line, "ok" if reply.endswith(": ok") else "fail", reply) + elif cmd in ("page", "hit", "stats", "ctl"): + # Query verbs: print the reply, never judge it. `ctl` is a raw + # passthrough for anything the channel grows later. + need_ctl(lineno, cmd) + reply = ctl.send(line if cmd != "ctl" else " ".join(args)) + record(lineno, line, "ok", reply) + elif cmd == "assert_page": + need_ctl(lineno, cmd) + want = " ".join(args) + got = ctl.send("page") + record(lineno, line, "ok" if got == want else "fail", + f"page is {got!r}" if got != want else "") + elif cmd == "assert_hit": + # assert_hit X Y CLASS [box=X1,Y1,WxH] [TEXT...]: a real tap at X,Y + # would land on an object of CLASS, with that exact bounding box if + # one is given, whose caption contains TEXT. This is the pre-tap + # self-check: "the button moved" fails HERE, by name, so it is + # never mistaken for the behaviour behind the button. + # + # The box is the strong identity. Two rows of a list share a class, + # and an icon's caption is a glyph nobody wants in a spec; the + # geometry the UI itself reports is what tells one instance from + # another. + need_ctl(lineno, cmd) + x, y, cls = args[0], args[1], args[2] + rest = args[3:] + want_box = None + if rest and rest[0].startswith("box="): + want_box = rest[0][4:] + rest = rest[1:] + want_text = " ".join(rest) + got = parse_hit(ctl.send(f"hit {x} {y}")) + if got is None: + record(lineno, line, "fail", "nothing clickable there") + elif got[0] != cls: + record(lineno, line, "fail", f"{got[0]} text={got[1]!r} box={got[2]}") + elif want_box and got[2] != want_box: + record(lineno, line, "fail", f"{cls} moved: box={got[2]}") + elif want_text and want_text not in got[1]: + record(lineno, line, "fail", f"{cls} text={got[1]!r}") + else: + record(lineno, line, "ok") else: sys.exit(f"FATAL: {script_path}:{lineno}: unknown command '{cmd}'") + crashed = ui_exited(console_path) + if crashed: + record(lineno, line, "fatal", crashed) + break + + results.close() + print(f"== {counts['ok']} ok, {counts['fail']} failed, {counts['fatal']} fatal " + f"-> {os.path.join(outdir, 'results.jsonl')}", flush=True) + if counts["fail"] or counts["fatal"]: + sys.exit(1) + def main(): if len(sys.argv) < 3: @@ -150,6 +324,8 @@ def main(): size = 720 if "--size" in sys.argv: size = int(sys.argv[sys.argv.index("--size") + 1]) + ctl_path = sys.argv[sys.argv.index("--ctl") + 1] if "--ctl" in sys.argv else None + console_path = sys.argv[sys.argv.index("--console") + 1] if "--console" in sys.argv else None s = socket.socket(socket.AF_UNIX) s.connect(path) @@ -162,7 +338,7 @@ def main(): elif cmd == "tap": do_tap(s, f, int(sys.argv[3]), int(sys.argv[4])) elif cmd == "drive": - drive(s, f, sys.argv[3], sys.argv[4], size) + drive(s, f, sys.argv[3], sys.argv[4], size, ctl_path, console_path) elif cmd == "quit": s.sendall(b'{"execute":"quit"}\n') diff --git a/qemu/tests/ui-drive.sh b/qemu/tests/ui-drive.sh index c718a1a..93bcb45 100755 --- a/qemu/tests/ui-drive.sh +++ b/qemu/tests/ui-drive.sh @@ -64,6 +64,7 @@ for _attempt in 1 2 3; do PORT=$((21000 + RANDOM % 20000)) : > "$WORK/console.log" bash "$QDIR/run.sh" --kernel "$ZIMAGE" --display headless --qmp "$WORK/qmp.sock" \ + --ctl "$WORK/ctl.sock" \ --ssh-port "$PORT" --http-port $((PORT + 1)) --api-port $((PORT + 2)) \ > "$WORK/console.log" 2>&1 & QEMU_PID=$! @@ -118,15 +119,43 @@ done exit 1 } -echo "== driving $SCRIPT" -python3 "$HERE/qmp.py" "$WORK/qmp.sock" drive "$SCRIPT" "$OUTDIR" +# The control bridge (init -> warden-ui's debug FIFO, see run.sh --ctl) comes +# up with the UI; a script's first `page`/`hit` must not race it. Bounded, and +# fail-closed: a scenario that asserts on UI state needs the channel, and a +# silently absent one would turn every assertion into an infrastructure error +# dressed as a test result. +echo "== waiting for the control bridge" +deadline=$((SECONDS + 60)) +until grep -aq 'init: control bridge on' "$WORK/console.log"; do + [ $SECONDS -lt $deadline ] || { + echo "FATAL: the control bridge never announced itself (run.sh --ctl / init marker)" >&2 + tail -25 "$WORK/console.log" >&2 + exit 1 + } + sleep 1 +done -# The UI must still be alive: see the header. Checked AFTER the script so a -# crash caused by the interaction is caught, which is the usual case. +echo "== driving $SCRIPT" +# Every step lands in results.jsonl (ok / fail / fatal); an assertion mismatch +# is a `fail` and the run continues, so one run reports every broken +# expectation. The driver's exit status is the verdict; capture it rather than +# let `set -e` skip the backstop and the summary below. +drive_rc=0 +python3 "$HERE/qmp.py" "$WORK/qmp.sock" drive "$SCRIPT" "$OUTDIR" \ + --ctl "$WORK/ctl.sock" --console "$WORK/console.log" || drive_rc=$? + +# The UI must still be alive: see the header. The driver checks this after +# every step and pins a crash to the step that caused it; this is the +# backstop for a death after the last step, or a driver that itself fell over. if grep -aq 'warden-ui EXITED' "$WORK/console.log"; then echo "FATAL: warden-ui DIED during the run:" >&2 grep -a -A22 'warden-ui EXITED' "$WORK/console.log" >&2 exit 1 fi +if [ "$drive_rc" -ne 0 ]; then + echo "UI-DRIVE-FAIL: see $OUTDIR/results.jsonl" >&2 + grep -E '"status": "(fail|fatal)"' "$OUTDIR/results.jsonl" >&2 || true + exit 1 +fi -echo "UI-DRIVE-PASS (screenshots in $OUTDIR)" +echo "UI-DRIVE-PASS (results in $OUTDIR/results.jsonl, screenshots in $OUTDIR)"