qemu: review pass over the rig driver and boot script

Four review passes with fixes between them (flare-edge's flow-framework
review, 2026-09-09). qmp.py: drive() split out of a 330-line dispatcher,
every verb guarded so a raising verb records a fatal row instead of ending
the run, the shot path sanitised, the rs485 and wait verbs judged through
shared helpers; imgtools.py: a bench subcommand for phash/structural
timings and a colour probe that samples instead of scanning the frame;
ui-drive.sh: the boot poll no longer walks every pixel per tick and the
simulator's control socket path is passed as one word. Offline tests: 8.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N3G6m9Aw5RyVY4ZowtKzEj
This commit is contained in:
Noah
2026-09-09 13:52:47 -06:00
co-authored by Claude Fable 5.1
parent 48be35dc7b
commit 8805b6106c
4 changed files with 1037 additions and 342 deletions
+492 -316
View File
@@ -43,11 +43,16 @@ a real panel, so these verbs mean the same thing on the rig and on hardware:
--rs485-control (ui-drive.sh --rs485-devices)
ctl WORDS... raw passthrough for anything the channel grows
assert_page MENU/TAB the active page is exactly this
assert_hit X Y CLASS [TEXT...]
a real tap at X,Y lands on an object of CLASS whose
assert_hit X Y CLASS [box=X1,Y1,WxH] [TEXT...]
a real tap at X,Y lands on an object of CLASS,
with that exact box when one is given, whose
caption contains TEXT: the pre-tap self-check that
reports a MOVED button as such, not as broken
behaviour
wait_hit X Y CLASS TIMEOUT_S [box=X1,Y1,WxH] [TEXT...]
assert_hit, polled every 0.5s (poll_until) until it
holds or TIMEOUT_S elapses -- for a page still
building when the check runs
Two more channels reach behavioural state instead of the widget tree.
`--refs FILE` (default <outdir>/refs.json) is the reference store the pixel
@@ -85,6 +90,9 @@ math, this file only drives it.
compare against its stored hash at its stored
tolerance. TOLERANCE, when given, must be the one
NAME was captured under (flowc.py always emits it).
wait_region NAME TOLERANCE TIMEOUT_S
assert_region, polled every 0.5s (poll_until) until
it holds or TIMEOUT_S elapses
assert_ocr NAME REGEX same crop, run through `tesseract`, REGEX searched
against the extracted text (spaces in REGEX are
fine, same as assert_hit's TEXT).
@@ -288,14 +296,25 @@ def judge_hit(ctl, x, y, cls, rest):
def poll_until(check, timeout_s, period=0.5):
"""Run CHECK (-> (ok, detail)) until it holds or TIMEOUT_S passes;
-> (ok, detail, seconds waited). The wait_* verbs share this so they
all mean the same thing by a timeout."""
-> (ok, detail, seconds waited, fatal). The wait_* verbs share this so
they all mean the same thing by a timeout.
A CHECK that raises RuntimeError or OSError -- the control channel
dying mid-poll, e.g. a socket.timeout past Ctl's own 15s per-call
timeout -- stops polling right there instead of retrying a channel
that is probably gone, and comes back with FATAL true so the caller
records a `fatal` row (the same idea as ConsoleWatch.check() turning a
console crash into a labeled row) rather than the exception escaping
drive() and silently dropping every step after it."""
start = time.monotonic()
deadline = start + timeout_s
while True:
ok, detail = check()
try:
ok, detail = check()
except (RuntimeError, OSError) as e:
return False, str(e), time.monotonic() - start, True
if ok or time.monotonic() >= deadline:
return ok, detail, time.monotonic() - start
return ok, detail, time.monotonic() - start, False
time.sleep(period)
@@ -311,21 +330,52 @@ def parse_hit(reply):
return m.group(1), m.group(2) or "", box
def ui_exited(console_path):
"""The stage-2 init announces a dead UI on the console; the framebuffer
does not, it just keeps the last frame. Checked after EVERY step so a crash
is pinned to the step that caused it, not discovered at the end."""
if not console_path:
class ConsoleWatch:
"""Incremental check for the stage-2 init's console crash marker.
The framebuffer just keeps the last frame on a crash; only the console
announces it, and drive() checks after EVERY step so a crash is pinned
to the step that caused it, not discovered at the end. But the console
log is not a static boot log -- rootfs/sbin/init pipes warden-ui's own
stdout through `tee` into the same stream the host captures, so it grows
for the life of the run. A fresh open()+read() of the WHOLE file on
every one of those per-step checks costs O(steps * final_size) for
nothing: a marker absent from bytes already scanned cannot retroactively
appear there. This instead keeps the byte offset already scanned and
reads only what was appended since the last check, with a short tail
kept across calls so a marker split across two reads is still caught.
One instance per drive() run."""
MARKER = b"warden-ui EXITED"
def __init__(self, console_path):
self.path = console_path
self.offset = 0
self.tail = b""
self.found = None
def check(self):
"""-> the matched line (decoded), once and cached forever after
(nothing later needs another read); None while nothing has matched
yet, or PATH is unset or unreadable."""
if not self.path or self.found is not None:
return self.found
try:
with open(self.path, "rb") as fh:
fh.seek(self.offset)
chunk = fh.read()
self.offset = fh.tell()
except OSError:
return None
data = self.tail + chunk
i = data.find(self.MARKER)
if i >= 0:
self.found = data[i:i + 80].split(b"\n", 1)[0].decode("utf-8", "replace")
return self.found
# Keep enough of the tail that a marker whose first byte landed in
# this read, but the rest in the next one, is still caught.
self.tail = data[-(len(self.MARKER) - 1):]
return None
try:
with open(console_path, "rb") as fh:
data = fh.read()
except OSError:
return None
i = data.find(b"warden-ui EXITED")
if i < 0:
return None
return data[i:i + 80].split(b"\n", 1)[0].decode("utf-8", "replace")
PATH_TOKEN_RE = re.compile(r'^([^\[\]]*)((?:\[\d+\])*)$')
@@ -409,9 +459,14 @@ def apply_op(op, actual, raw_value):
def fetch_status_json(ctl):
"""`@cat` the guest's webstatus.c snapshot over the control channel and
parse it. -> (doc, None) or (None, detail); never raises, so a torn read
or a not-yet-written file is an ordinary retry/fail for the caller, not a
crash of the whole driver."""
parse it. -> (doc, None) or (None, detail) for a torn read or a
not-yet-written file -- an ordinary retry/fail for the caller, not a
crash of the whole driver. ctl.send() itself is deliberately NOT guarded
here: a dead control channel (RuntimeError on EOF, OSError/socket.timeout
on a stalled read) is left to propagate, so wait_json's poll_until (and
drive()'s own top-level guard for assert_json) can tell a dead channel
apart from a value the document simply doesn't have yet, and report the
former as a fast fatal rather than retrying it out to TIMEOUT_S."""
reply = ctl.send(f"@cat {STATUS_JSON_PATH}")
if reply.startswith("bridge: no such file"):
return None, reply
@@ -490,6 +545,23 @@ def write_png(img, path):
_PILImage.frombytes("RGB", (w, h), data).save(path)
def safe_out_path(outdir, name, prefix="", suffix=""):
"""<outdir>/PREFIXnameSUFFIX, or (None, detail) for a NAME that could
write outside outdir. flowc.py does not restrict shot/region/ocr names
beyond forbidding whitespace (see its own `shot` and `one_token`
comments), so a hand-edited or buggy *.drive.txt line -- or a refs.json
entry with a name capture_region itself would never have written -- must
not be allowed to place NAME ahead of outdir in the joined path. An
absolute NAME (e.g. `shot /etc/cron.d/x`) makes os.path.join() discard
outdir entirely and return NAME verbatim; a NAME with any other '/' can
still walk out of outdir with enough '..' segments. Mirrors
flow-run-hw.sh's shot_out_path() for the identical class of bug on the
hardware runner."""
if not name or "/" in name:
return None, f"{name!r} must not contain '/' (would escape outdir)"
return os.path.join(os.path.abspath(outdir), f"{prefix}{name}{suffix}"), None
def rs485_send(control_path, line):
"""One command to mbsim.py's control socket (flare-edge tools/modbus-sim,
--control): -> its one-line reply, `ok ...` or `error ...`."""
@@ -500,6 +572,382 @@ def rs485_send(control_path, line):
return c.makefile("r").readline().strip()
class Ctx:
"""State one drive() run threads through every verb handler below,
replacing the closures drive() used to build fresh on each call (record
stays in drive() itself -- it owns the results.jsonl handle -- but
need_ctl, need_imgtools and fresh_region move here since verb handlers,
not drive(), are what call them now). One instance per run."""
def __init__(self, s, f, size, ctl, refs, refs_path, rs485_control, outdir, script_path):
self.s = s
self.f = f
self.size = size
self.ctl = ctl
self.refs = refs
self.refs_path = refs_path
self.rs485_control = rs485_control
self.outdir = outdir
self.script_path = script_path
def need_ctl(self, lineno, cmd):
if self.ctl is None:
sys.exit(f"FATAL: {self.script_path}:{lineno}: '{cmd}' needs the control "
f"channel; run with --ctl (ui-drive.sh passes it)")
def need_imgtools(self, lineno, cmd):
# A whole-script infrastructure gap (imgtools.py absent), same class
# as a missing --ctl: no region/ocr verb can do anything without it,
# so this exits the process rather than recording a per-step fail.
if imgtools is None:
sys.exit(f"FATAL: {self.script_path}:{lineno}: '{cmd}' needs imgtools.py "
f"next to qmp.py (vendor/warden-sdk/qemu/tests/imgtools.py)")
def fresh_region(self, name, tag):
"""screendump NOW (never reuse an earlier `shot`) and crop to NAME's
stored box. Fails closed on an uncaptured NAME: assert_region and
assert_ocr both need this, capture_region does not."""
ref = self.refs.get(name)
if ref is None:
return None, None, f"no reference for {name!r} (run capture_region first)"
path, err = safe_out_path(self.outdir, name, prefix=f"{tag}_", suffix=".ppm")
if err:
return None, None, err
rpc(self.s, self.f, {"execute": "screendump", "arguments": {"filename": path}})
try:
img = imgtools.load_ppm(path)
cropped = imgtools.crop(img, ref["x"], ref["y"], ref["w"], ref["h"])
except (ValueError, OSError, KeyError, TypeError) as e:
# A box outside this screendump (captured at another --size, or
# mistyped in the store) is a per-step fatal, never a traceback
# that ends the run: every later step still gets judged.
return None, None, f"reference {name!r} unusable: {e}"
return ref, cropped, None
# One function per script verb, VERBS-dispatched below instead of a single
# growing if/elif chain: each takes the run's shared Ctx plus the parsed
# line and returns either None (no results.jsonl row -- only echo does
# this) or (status, detail) for drive() to record. Keeping one function per
# verb, the way flowc.py tables its own CHANNELS/ACTIONS vocabulary, is
# what lets a verb be read, tested or reused on its own instead of only as
# one arm of drive().
def verb_shot(ctx, lineno, cmd, args, line):
name = args[0]
path, err = safe_out_path(ctx.outdir, name, suffix=".ppm")
if err:
return "fail", f"shot name {err}"
rpc(ctx.s, ctx.f, {"execute": "screendump", "arguments": {"filename": path}})
return "ok", ""
def verb_tap(ctx, lineno, cmd, args, line):
x, y = int(args[0]), int(args[1])
do_tap(ctx.s, ctx.f, to_axis(x, ctx.size), to_axis(y, ctx.size))
return "ok", ""
def verb_swipe(ctx, lineno, cmd, args, line):
ms = int(args[4]) if len(args) > 4 else 400
do_swipe(ctx.s, ctx.f, int(args[0]), int(args[1]), int(args[2]), int(args[3]), ctx.size, ms)
return "ok", ""
def verb_fling(ctx, lineno, cmd, args, line):
do_swipe(ctx.s, ctx.f, int(args[0]), int(args[1]), int(args[2]), int(args[3]),
ctx.size, ms=120)
return "ok", ""
def verb_sleep(ctx, lineno, cmd, args, line):
time.sleep(float(args[0]))
return "ok", ""
def verb_echo(ctx, lineno, cmd, args, line):
print(f" [{lineno}] {' '.join(args)}", flush=True)
return None
def verb_nav(ctx, lineno, cmd, args, line):
ctx.need_ctl(lineno, cmd)
reply = ctx.ctl.send("nav " + " ".join(args))
return ("ok" if reply.endswith(": ok") else "fail"), reply
def verb_wake(ctx, lineno, cmd, args, line):
# Judged, unlike the query verbs: a UI that does not answer the wake is
# one whose next tap may be swallowed, and that must not read as a
# passing step.
ctx.need_ctl(lineno, cmd)
reply = ctx.ctl.send("wake")
return ("ok" if reply == "wake: ok" else "fail"), reply
def verb_scroll_home(ctx, lineno, cmd, args, line):
# scroll X Y DY: the scrollable under the pixel moves DY pixels with no
# animation (warden_debug.c), the deterministic stand-in for a swipe
# whose only job was to bring a control into view.
# home: close every popup and return to Dashboard/Dashboard, the state a
# fresh boot starts in, so a live panel can run one flow after another.
# Both are judged by the channel's own ack.
ctx.need_ctl(lineno, cmd)
reply = ctx.ctl.send(line)
return ("ok" if reply.startswith(f"{cmd}: ok") else "fail"), reply
def verb_query(ctx, lineno, cmd, args, line):
# page | hit | stats | ctl: print the reply, never judge it. `ctl` is a
# raw passthrough for anything the channel grows later.
ctx.need_ctl(lineno, cmd)
reply = ctx.ctl.send(line if cmd != "ctl" else " ".join(args))
return "ok", reply
def verb_rs485(ctx, lineno, cmd, args, line):
# rs485 silence|restore ADDR: take a simulated device off the bus (it
# holds its address and answers nothing, exactly an absent unit) or put
# it back, while the guest keeps polling. This is the only runtime lever
# on the roster ui-drive.sh --rs485-devices fixed at boot, and it exists
# so a flow can prove a screen noticing an adopted device go quiet
# (flare-edge #184). No simulated bus in this run is fatal: the step
# could not check anything, and a silent skip would read as a pass.
if ctx.rs485_control is None:
return "fatal", ("no simulated RS485 bus in this run "
"(ui-drive.sh --rs485-devices)")
if len(args) != 2 or args[0] not in ("silence", "restore"):
return "fatal", "expected: rs485 silence|restore ADDR"
try:
reply = rs485_send(ctx.rs485_control, f"{args[0]} {args[1]}")
except OSError as e:
reply = f"error control socket: {e}"
return ("ok" if reply.startswith("ok") else "fail"), reply
def verb_assert_page(ctx, lineno, cmd, args, line):
ctx.need_ctl(lineno, cmd)
want = " ".join(args)
got = ctx.ctl.send("page")
return ("ok" if got == want else "fail"), (f"page is {got!r}" if got != want else "")
def verb_assert_hit(ctx, lineno, cmd, args, line):
# assert_hit X Y CLASS [box=X1,Y1,WxH] [TEXT...]: a real tap at X,Y
# would land on an object of CLASS, with that exact bounding box if one
# is given, whose caption contains TEXT. This is the pre-tap self-check:
# "the button moved" fails HERE, by name, so it is never mistaken for
# the behaviour behind the button.
#
# The box is the strong identity. Two rows of a list share a class, and
# an icon's caption is a glyph nobody wants in a spec; the geometry the
# UI itself reports is what tells one instance from another.
ctx.need_ctl(lineno, cmd)
ok, detail = judge_hit(ctx.ctl, args[0], args[1], args[2], args[3:])
return ("ok" if ok else "fail"), detail
def verb_wait_hit(ctx, lineno, cmd, args, line):
# wait_hit X Y CLASS TIMEOUT_S [box=X1,Y1,WxH] [TEXT...]: assert_hit
# polled every 0.5s until it holds or TIMEOUT_S passes. A page still
# building after `nav` answers `hit` with whatever is there at that
# instant, so a check one second in raced the layout (flare-edge #175).
# The timeout is the author's bound on how long settling may take, not a
# sleep: a check that holds early returns early, and the detail says how
# long it took.
ctx.need_ctl(lineno, cmd)
x, y, cls, timeout_s = args[0], args[1], args[2], float(args[3])
rest = args[4:]
ok, detail, waited, fatal = poll_until(
lambda: judge_hit(ctx.ctl, x, y, cls, rest), timeout_s)
if fatal:
return "fatal", detail
return ("ok" if ok else "fail"), (f"waited {waited:.1f}s" if ok else detail)
def verb_assert_json(ctx, lineno, cmd, args, line):
# assert_json PATH OP VALUE: one read of the guest's status JSON (see
# eval_json). PATH not (yet) in the doc is a `fail`, since the doc is a
# live snapshot, not a schema.
ctx.need_ctl(lineno, cmd)
ok, detail = eval_json(ctx.ctl, args[0], args[1], args[2])
return ("ok" if ok else "fail"), detail
def verb_wait_json(ctx, lineno, cmd, args, line):
# wait_json PATH OP VALUE TIMEOUT_S: eval_json polled every 0.5s
# (poll_until) until it holds or the deadline passes. The status file is
# written by a 2s guest timer (webstatus.c), so this exists because a
# value the doc will reach shortly is not the same fact as a value it
# never reaches -- assert_json alone cannot tell those apart.
ctx.need_ctl(lineno, cmd)
timeout_s = float(args[3])
ok, detail, waited, fatal = poll_until(
lambda: eval_json(ctx.ctl, args[0], args[1], args[2]), timeout_s)
if fatal:
return "fatal", detail
return ("ok" if ok else "fail"), (f"waited {waited:.1f}s" if ok else detail)
def verb_assert_stat(ctx, lineno, cmd, args, line):
# assert_stat FIELD OP VALUE: FIELD off a fresh `stats` reply.
ctx.need_ctl(lineno, cmd)
field, op, value = args[0], args[1], args[2]
if field not in ("cpu", "fps", "render", "idle"):
return "fail", (f"unknown stat field {field!r} "
f"(known: cpu, fps, render, idle)")
stats = parse_stats(ctx.ctl.send("stats"))
if field not in stats:
return "fail", "no such field"
try:
ok = apply_op(op, stats[field], value)
except (TypeError, ValueError) as e:
return "fail", str(e)
return ("ok" if ok else "fail"), ("" if ok else f"{field} is {stats[field]}")
def verb_capture_region(ctx, lineno, cmd, args, line):
# capture_region NAME X Y W H TOLERANCE: (re)writes NAME in the refs
# file from a fresh screendump. Both hashes are stored regardless of
# TOLERANCE so a later spec edit can change tolerance class without a
# recapture.
ctx.need_imgtools(lineno, cmd)
name = args[0]
x, y, w, h = int(args[1]), int(args[2]), int(args[3]), int(args[4])
tolerance = args[5]
if tolerance not in ("exact", "loose", "structural"):
return "fail", (f"unknown tolerance {tolerance!r} "
f"(known: exact, loose, structural)")
path, err = safe_out_path(ctx.outdir, name, prefix="capture_", suffix=".ppm")
if err:
return "fail", f"region name {err}"
rpc(ctx.s, ctx.f, {"execute": "screendump", "arguments": {"filename": path}})
try:
cropped = imgtools.crop(imgtools.load_ppm(path), x, y, w, h)
except (ValueError, OSError) as e:
return "fatal", f"cannot capture {name!r}: {e}"
ctx.refs[name] = {
"x": x, "y": y, "w": w, "h": h, "tolerance": tolerance,
"phash": f"{imgtools.phash(cropped):016x}",
"structural": imgtools.structural(cropped).hex(),
}
save_refs(ctx.refs_path, ctx.refs)
return "ok", f"captured box={x},{y},{w}x{h}"
def verb_assert_region(ctx, lineno, cmd, args, line):
# assert_region NAME [TOLERANCE]: fresh screendump, crop to NAME's
# stored box, compare at NAME's stored tolerance. A missing NAME is
# FATAL for this step: an uncaptured reference is a spec/authoring gap,
# not a UI defect the run should merely `fail` on. So is a TOLERANCE
# other than the one NAME was captured under: the script and the
# reference would be two claims about the same pixels, and judging by
# either alone would hide that.
ctx.need_imgtools(lineno, cmd)
name = args[0]
want_tol = args[1] if len(args) > 1 else None
ref = ctx.refs.get(name)
if ref is not None and want_tol and want_tol != ref.get("tolerance"):
return "fatal", (f"reference {name!r} was captured as {ref.get('tolerance')}, "
f"script expects {want_tol}: recapture or fix the spec")
ref, cropped, err = ctx.fresh_region(name, "assert")
if err:
return "fatal", err
ok, detail = imgtools.compare(ref, cropped, ref["tolerance"])
return ("ok" if ok else "fail"), detail
def verb_wait_region(ctx, lineno, cmd, args, line):
# wait_region NAME TOLERANCE TIMEOUT_S: assert_region polled every 0.5s,
# a fresh screendump each time, until the crop matches or the deadline
# passes (flare-edge #175). A missing reference or a foreign tolerance
# is fatal exactly as for assert_region: waiting cannot fix either, so
# polling stops on the first such answer.
ctx.need_imgtools(lineno, cmd)
name, want_tol, timeout_s = args[0], args[1], float(args[2])
ref = ctx.refs.get(name)
if ref is not None and want_tol != ref.get("tolerance"):
return "fatal", (f"reference {name!r} was captured as {ref.get('tolerance')}, "
f"script expects {want_tol}: recapture or fix the spec")
fault = []
def check_region():
r, cropped, err = ctx.fresh_region(name, "assert")
if err:
fault.append(err)
return True, err
return imgtools.compare(r, cropped, r["tolerance"])
ok, detail, waited, poll_fatal = poll_until(check_region, timeout_s)
if fault:
return "fatal", fault[0]
if poll_fatal:
return "fatal", detail
return ("ok" if ok else "fail"), (f"waited {waited:.1f}s" if ok else detail)
def verb_assert_ocr(ctx, lineno, cmd, args, line):
# assert_ocr NAME REGEX: same crop as assert_region, OCR'd through
# `tesseract`, REGEX searched (re.search, spaces allowed like
# assert_hit's TEXT) against the extracted text. No tesseract on PATH,
# or no reference for NAME, is FATAL: pixels were never actually
# checked, so this must never look like a skipped-but-passing step.
ctx.need_imgtools(lineno, cmd)
name, pattern = args[0], " ".join(args[1:])
if shutil.which("tesseract") is None:
return "fatal", "tesseract not installed"
ref, cropped, err = ctx.fresh_region(name, "ocr")
if err:
return "fatal", err
# fresh_region() above already ran NAME through safe_out_path() and
# bailed on `err` if it could escape outdir, so it is safe to build this
# second path from NAME directly.
png_path = os.path.join(os.path.abspath(ctx.outdir), f"ocr_{name}.png")
try:
write_png(cropped, png_path)
text = subprocess.run(
["tesseract", png_path, "stdout"],
capture_output=True, text=True, timeout=20, check=True,
).stdout
except Exception as e:
return "fatal", f"ocr failed: {e}"
if re.search(pattern, text):
return "ok", ""
return "fail", f"text was {text.strip()!r}"
# verb NAME -> handler. Several names share one handler (scroll/home;
# page/hit/stats/ctl) the same way they shared one elif arm before; the
# handler still gets the matched name as `cmd` for messages that name it.
VERBS = {
"shot": verb_shot,
"tap": verb_tap,
"swipe": verb_swipe,
"fling": verb_fling,
"sleep": verb_sleep,
"echo": verb_echo,
"nav": verb_nav,
"wake": verb_wake,
"scroll": verb_scroll_home,
"home": verb_scroll_home,
"page": verb_query,
"hit": verb_query,
"stats": verb_query,
"ctl": verb_query,
"rs485": verb_rs485,
"assert_page": verb_assert_page,
"assert_hit": verb_assert_hit,
"wait_hit": verb_wait_hit,
"assert_json": verb_assert_json,
"wait_json": verb_wait_json,
"assert_stat": verb_assert_stat,
"capture_region": verb_capture_region,
"assert_region": verb_assert_region,
"wait_region": verb_wait_region,
"assert_ocr": verb_assert_ocr,
}
def drive(s, f, script_path, outdir, size, ctl_path=None, console_path=None, refs_path=None,
rs485_control=None):
os.makedirs(outdir, exist_ok=True)
@@ -518,40 +966,9 @@ def drive(s, f, script_path, outdir, size, ctl_path=None, console_path=None, ref
tag = {"ok": "", "fail": "FAIL ", "fatal": "FATAL "}[status]
print(f" [{lineno}] {tag}{cmd}{(' -- ' + detail) if detail else ''}", flush=True)
def need_ctl(lineno, cmd):
if ctl is None:
sys.exit(f"FATAL: {script_path}:{lineno}: '{cmd}' needs the control "
f"channel; run with --ctl (ui-drive.sh passes it)")
def need_imgtools(lineno, cmd):
# A whole-script infrastructure gap (imgtools.py absent), same class
# as a missing --ctl: no region/ocr verb can do anything without it,
# so this exits the process rather than recording a per-step fail.
if imgtools is None:
sys.exit(f"FATAL: {script_path}:{lineno}: '{cmd}' needs imgtools.py "
f"next to qmp.py (vendor/warden-sdk/qemu/tests/imgtools.py)")
refs_path = refs_path or os.path.join(outdir, "refs.json")
refs = load_refs(refs_path)
def fresh_region(name, tag):
"""screendump NOW (never reuse an earlier `shot`) and crop to NAME's
stored box. Fails closed on an uncaptured NAME: assert_region and
assert_ocr both need this, capture_region does not."""
ref = refs.get(name)
if ref is None:
return None, None, f"no reference for {name!r} (run capture_region first)"
path = os.path.join(os.path.abspath(outdir), f"{tag}_{name}.ppm")
rpc(s, f, {"execute": "screendump", "arguments": {"filename": path}})
try:
img = imgtools.load_ppm(path)
cropped = imgtools.crop(img, ref["x"], ref["y"], ref["w"], ref["h"])
except (ValueError, OSError, KeyError, TypeError) as e:
# A box outside this screendump (captured at another --size, or
# mistyped in the store) is a per-step fatal, never a traceback
# that ends the run: every later step still gets judged.
return None, None, f"reference {name!r} unusable: {e}"
return ref, cropped, None
ctx = Ctx(s, f, size, ctl, load_refs(refs_path), refs_path, rs485_control, outdir, script_path)
console = ConsoleWatch(console_path)
for lineno, raw in enumerate(lines, 1):
line = raw.split("#", 1)[0].strip()
@@ -560,269 +977,28 @@ def drive(s, f, script_path, outdir, size, ctl_path=None, console_path=None, ref
parts = line.split()
cmd, args = parts[0], parts[1:]
if cmd == "shot":
name = args[0]
path = os.path.join(os.path.abspath(outdir), f"{name}.ppm")
rpc(s, f, {"execute": "screendump", "arguments": {"filename": path}})
record(lineno, line, "ok")
elif cmd == "tap":
x, y = int(args[0]), int(args[1])
do_tap(s, f, to_axis(x, size), to_axis(y, size))
record(lineno, line, "ok")
elif cmd == "swipe":
ms = int(args[4]) if len(args) > 4 else 400
do_swipe(s, f, int(args[0]), int(args[1]), int(args[2]), int(args[3]), size, ms)
record(lineno, line, "ok")
elif cmd == "fling":
do_swipe(s, f, int(args[0]), int(args[1]), int(args[2]), int(args[3]),
size, ms=120)
record(lineno, line, "ok")
elif cmd == "sleep":
time.sleep(float(args[0]))
record(lineno, line, "ok")
elif cmd == "echo":
print(f" [{lineno}] {' '.join(args)}", flush=True)
elif cmd == "nav":
need_ctl(lineno, cmd)
reply = ctl.send("nav " + " ".join(args))
record(lineno, line, "ok" if reply.endswith(": ok") else "fail", reply)
elif cmd == "wake":
# Judged, unlike the query verbs: a UI that does not answer the
# wake is one whose next tap may be swallowed, and that must not
# read as a passing step.
need_ctl(lineno, cmd)
reply = ctl.send("wake")
record(lineno, line, "ok" if reply == "wake: ok" else "fail", reply)
elif cmd in ("scroll", "home"):
# scroll X Y DY: the scrollable under the pixel moves DY pixels
# with no animation (warden_debug.c), the deterministic stand-in
# for a swipe whose only job was to bring a control into view.
# home: close every popup and return to Dashboard/Dashboard, the
# state a fresh boot starts in, so a live panel can run one flow
# after another. Both are judged by the channel's own ack.
need_ctl(lineno, cmd)
reply = ctl.send(line)
record(lineno, line, "ok" if reply.startswith(f"{cmd}: ok") else "fail", reply)
elif cmd in ("page", "hit", "stats", "ctl"):
# Query verbs: print the reply, never judge it. `ctl` is a raw
# passthrough for anything the channel grows later.
need_ctl(lineno, cmd)
reply = ctl.send(line if cmd != "ctl" else " ".join(args))
record(lineno, line, "ok", reply)
elif cmd == "rs485":
# rs485 silence|restore ADDR: take a simulated device off the bus
# (it holds its address and answers nothing, exactly an absent
# unit) or put it back, while the guest keeps polling. This is
# the only runtime lever on the roster ui-drive.sh --rs485-devices
# fixed at boot, and it exists so a flow can prove a screen
# noticing an adopted device go quiet (flare-edge #184). No
# simulated bus in this run is fatal: the step could not check
# anything, and a silent skip would read as a pass.
if rs485_control is None:
record(lineno, line, "fatal", "no simulated RS485 bus in this run "
"(ui-drive.sh --rs485-devices)")
elif len(args) != 2 or args[0] not in ("silence", "restore"):
record(lineno, line, "fatal", "expected: rs485 silence|restore ADDR")
else:
try:
reply = rs485_send(rs485_control, f"{args[0]} {args[1]}")
except OSError as e:
reply = f"error control socket: {e}"
record(lineno, line, "ok" if reply.startswith("ok") else "fail", reply)
elif cmd == "assert_page":
need_ctl(lineno, cmd)
want = " ".join(args)
got = ctl.send("page")
record(lineno, line, "ok" if got == want else "fail",
f"page is {got!r}" if got != want else "")
elif cmd == "assert_hit":
# assert_hit X Y CLASS [box=X1,Y1,WxH] [TEXT...]: a real tap at X,Y
# would land on an object of CLASS, with that exact bounding box if
# one is given, whose caption contains TEXT. This is the pre-tap
# self-check: "the button moved" fails HERE, by name, so it is
# never mistaken for the behaviour behind the button.
#
# The box is the strong identity. Two rows of a list share a class,
# and an icon's caption is a glyph nobody wants in a spec; the
# geometry the UI itself reports is what tells one instance from
# another.
need_ctl(lineno, cmd)
ok, detail = judge_hit(ctl, args[0], args[1], args[2], args[3:])
record(lineno, line, "ok" if ok else "fail", detail)
elif cmd == "wait_hit":
# wait_hit X Y CLASS TIMEOUT_S [box=X1,Y1,WxH] [TEXT...]: assert_hit
# polled every 0.5s until it holds or TIMEOUT_S passes. A page
# still building after `nav` answers `hit` with whatever is there
# at that instant, so a check one second in raced the layout
# (flare-edge #175). The timeout is the author's bound on how long
# settling may take, not a sleep: a check that holds early returns
# early, and the detail says how long it took.
need_ctl(lineno, cmd)
x, y, cls, timeout_s = args[0], args[1], args[2], float(args[3])
rest = args[4:]
ok, detail, waited = poll_until(lambda: judge_hit(ctl, x, y, cls, rest), timeout_s)
record(lineno, line, "ok" if ok else "fail",
f"waited {waited:.1f}s" if ok else detail)
elif cmd == "assert_json":
# assert_json PATH OP VALUE: one read of the guest's status JSON
# (see eval_json). PATH not (yet) in the doc is a `fail`, since
# the doc is a live snapshot, not a schema.
need_ctl(lineno, cmd)
ok, detail = eval_json(ctl, args[0], args[1], args[2])
record(lineno, line, "ok" if ok else "fail", detail)
elif cmd == "wait_json":
# wait_json PATH OP VALUE TIMEOUT_S: poll eval_json every 0.5s
# until it holds or the deadline passes. The status file is
# written by a 2s guest timer (webstatus.c), so this exists
# because a value the doc will reach shortly is not the same
# fact as a value it never reaches -- assert_json alone cannot
# tell those apart.
need_ctl(lineno, cmd)
timeout_s = float(args[3])
start = time.monotonic()
deadline = start + timeout_s
while True:
ok, detail = eval_json(ctl, args[0], args[1], args[2])
if ok or time.monotonic() >= deadline:
break
time.sleep(0.5)
waited = time.monotonic() - start
record(lineno, line, "ok" if ok else "fail",
f"waited {waited:.1f}s" if ok else detail)
elif cmd == "assert_stat":
# assert_stat FIELD OP VALUE: FIELD off a fresh `stats` reply.
need_ctl(lineno, cmd)
field, op, value = args[0], args[1], args[2]
if field not in ("cpu", "fps", "render", "idle"):
record(lineno, line, "fail", f"unknown stat field {field!r} "
f"(known: cpu, fps, render, idle)")
else:
stats = parse_stats(ctl.send("stats"))
if field not in stats:
record(lineno, line, "fail", "no such field")
else:
try:
ok = apply_op(op, stats[field], value)
except (TypeError, ValueError) as e:
record(lineno, line, "fail", str(e))
else:
record(lineno, line, "ok" if ok else "fail",
"" if ok else f"{field} is {stats[field]}")
elif cmd == "capture_region":
# capture_region NAME X Y W H TOLERANCE: (re)writes NAME in the
# refs file from a fresh screendump. Both hashes are stored
# regardless of TOLERANCE so a later spec edit can change
# tolerance class without a recapture.
need_imgtools(lineno, cmd)
name = args[0]
x, y, w, h = int(args[1]), int(args[2]), int(args[3]), int(args[4])
tolerance = args[5]
if tolerance not in ("exact", "loose", "structural"):
record(lineno, line, "fail", f"unknown tolerance {tolerance!r} "
f"(known: exact, loose, structural)")
else:
path = os.path.join(os.path.abspath(outdir), f"capture_{name}.ppm")
rpc(s, f, {"execute": "screendump", "arguments": {"filename": path}})
try:
cropped = imgtools.crop(imgtools.load_ppm(path), x, y, w, h)
except (ValueError, OSError) as e:
record(lineno, line, "fatal", f"cannot capture {name!r}: {e}")
else:
refs[name] = {
"x": x, "y": y, "w": w, "h": h, "tolerance": tolerance,
"phash": f"{imgtools.phash(cropped):016x}",
"structural": imgtools.structural(cropped).hex(),
}
save_refs(refs_path, refs)
record(lineno, line, "ok", f"captured box={x},{y},{w}x{h}")
elif cmd == "assert_region":
# assert_region NAME [TOLERANCE]: fresh screendump, crop to
# NAME's stored box, compare at NAME's stored tolerance. A
# missing NAME is FATAL for this step: an uncaptured reference
# is a spec/authoring gap, not a UI defect the run should merely
# `fail` on. So is a TOLERANCE other than the one NAME was
# captured under: the script and the reference would be two
# claims about the same pixels, and judging by either alone
# would hide that.
need_imgtools(lineno, cmd)
name = args[0]
want_tol = args[1] if len(args) > 1 else None
ref = refs.get(name)
if ref is not None and want_tol and want_tol != ref.get("tolerance"):
record(lineno, line, "fatal",
f"reference {name!r} was captured as {ref.get('tolerance')}, "
f"script expects {want_tol}: recapture or fix the spec")
else:
ref, cropped, err = fresh_region(name, "assert")
if err:
record(lineno, line, "fatal", err)
else:
ok, detail = imgtools.compare(ref, cropped, ref["tolerance"])
record(lineno, line, "ok" if ok else "fail", detail)
elif cmd == "wait_region":
# wait_region NAME TOLERANCE TIMEOUT_S: assert_region polled every
# 0.5s, a fresh screendump each time, until the crop matches or
# the deadline passes (flare-edge #175). A missing reference or a
# foreign tolerance is fatal exactly as for assert_region: waiting
# cannot fix either, so polling stops on the first such answer.
need_imgtools(lineno, cmd)
name, want_tol, timeout_s = args[0], args[1], float(args[2])
ref = refs.get(name)
if ref is not None and want_tol != ref.get("tolerance"):
record(lineno, line, "fatal",
f"reference {name!r} was captured as {ref.get('tolerance')}, "
f"script expects {want_tol}: recapture or fix the spec")
else:
fault = []
def check_region():
r, cropped, err = fresh_region(name, "assert")
if err:
fault.append(err)
return True, err
return imgtools.compare(r, cropped, r["tolerance"])
ok, detail, waited = poll_until(check_region, timeout_s)
if fault:
record(lineno, line, "fatal", fault[0])
else:
record(lineno, line, "ok" if ok else "fail",
f"waited {waited:.1f}s" if ok else detail)
elif cmd == "assert_ocr":
# assert_ocr NAME REGEX: same crop as assert_region, OCR'd
# through `tesseract`, REGEX searched (re.search, spaces allowed
# like assert_hit's TEXT) against the extracted text. No
# tesseract on PATH, or no reference for NAME, is FATAL: pixels
# were never actually checked, so this must never look like a
# skipped-but-passing step.
need_imgtools(lineno, cmd)
name, pattern = args[0], " ".join(args[1:])
if shutil.which("tesseract") is None:
record(lineno, line, "fatal", "tesseract not installed")
else:
ref, cropped, err = fresh_region(name, "ocr")
if err:
record(lineno, line, "fatal", err)
else:
png_path = os.path.join(os.path.abspath(outdir), f"ocr_{name}.png")
try:
write_png(cropped, png_path)
text = subprocess.run(
["tesseract", png_path, "stdout"],
capture_output=True, text=True, timeout=20, check=True,
).stdout
except Exception as e:
record(lineno, line, "fatal", f"ocr failed: {e}")
else:
if re.search(pattern, text):
record(lineno, line, "ok")
else:
record(lineno, line, "fail", f"text was {text.strip()!r}")
else:
handler = VERBS.get(cmd)
if handler is None:
results.close() # the rows before this one are still evidence
sys.exit(f"FATAL: {script_path}:{lineno}: unknown command '{cmd}'")
crashed = ui_exited(console_path)
try:
result = handler(ctx, lineno, cmd, args, line)
except (RuntimeError, OSError) as e:
# poll_until already turns this into a fatal row for the wait_*
# verbs; every other verb reaches ctl.send() (nav, wake,
# scroll/home, page/hit/stats/ctl, assert_page, assert_hit,
# assert_json, assert_stat) or the QMP socket via rpc()
# (tap/swipe/fling/shot, the region verbs) with no guard of its
# own, so without this a dying channel or a gone QMP socket ends
# the whole run as an unhandled traceback instead of one fatal
# step. Recorded the same as any other fatal row: the run keeps
# going past it.
result = ("fatal", str(e))
if result is not None:
record(lineno, line, *result)
crashed = console.check()
if crashed:
record(lineno, line, "fatal", crashed)
break