NoahandClaude Fable 5.1
86a9544dcc
- rootfs/sbin/init: the control bridge answers `@cat PATH` locally so the driver can read /tmp/warden-web-status.json out of the guest. That file has no trailing newline; the bridge adds one so the sentinel stays on its own line and the line-based reader never blocks. - tests/qmp.py: wait_json/assert_json (dotted paths, eq/ne/contains/ len_eq/len_ge/gt/lt), assert_stat off the FIFO's stats reply, capture_region, assert_region NAME [TOLERANCE] and assert_ocr. A tolerance other than the captured one, a reference box that does not fit the screendump, a missing reference or a missing tesseract is FATAL for that step and the run continues (flare-edge #147). - tests/imgtools.py: P6 reader, crop, perceptual and structural hashes, compare, with a self-test. - tests/test_qmp_drive.py: drive() with QMP and the control channel faked, pinning the per-step ok/fail/fatal contract. - mkimage.sh SEED_DIR and ui-drive.sh --seed/--refs: settings fixtures staged into userdata before warden-ui starts, and a reference store handed to the driver. - ci: the driver tests and the imgtools self-test run in qemu-tools. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013aHKWzT5EF86RFKRMtAv9n
208 lines
8.3 KiB
Bash
Executable File
208 lines
8.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Drive the LVGL UI through a scripted interaction and collect screenshots.
|
|
#
|
|
# ui-shot.sh proves touch reaches the UI in one tap; this is the same rig for
|
|
# work that needs a SEQUENCE (swipe through the app rows, open a submenu, tap a
|
|
# tab, bring up the keyboard) with a screendump wherever the script asks for
|
|
# one. One boot serves the whole script, because booting per step (TCG, no KVM)
|
|
# costs about a minute and a real interaction is thirty steps.
|
|
#
|
|
# It also FAILS on a UI that died mid-script. The framebuffer keeps its last
|
|
# frame when warden-ui crashes, so screendumps carry on returning a plausible
|
|
# picture of a program that no longer exists; stage-2 init announces the exit on
|
|
# the console (see rootfs/sbin/init) and this greps for it after the run.
|
|
#
|
|
# FAILS CLOSED on missing prerequisites.
|
|
#
|
|
# Usage: ui-drive.sh [--seed FILE] [--refs FILE] <zImage-virt> <script> [out-dir]
|
|
# <script> is a qmp.py `drive` script: see its docstring for the commands.
|
|
# --seed FILE a seed manifest (flat YAML key: value, see flare-edge
|
|
# tools/seed-fixtures.py) run through that tool and staged
|
|
# into userdata/warden BEFORE boot, via mkimage.sh's SEED_DIR
|
|
# hook -- so a screen's first read at startup already sees
|
|
# it, not a value written after the race is already lost.
|
|
# Needs FLARE_EDGE=<checkout> to find the tool (same
|
|
# convention as ota-apply.sh / portal-scenario.sh).
|
|
# --refs FILE passed straight through to qmp.py drive's --refs (the
|
|
# region/ocr reference JSON); see qmp.py's own docstring.
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # qemu/tests/
|
|
QDIR="$(cd "$HERE/.." && pwd)" # qemu/
|
|
|
|
SEED_FILE=""
|
|
REFS_FILE=""
|
|
ARGS=()
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--seed) SEED_FILE="${2:?--seed needs a manifest path}"; shift 2 ;;
|
|
--refs) REFS_FILE="${2:?--refs needs a path}"; shift 2 ;;
|
|
--) shift; ARGS+=("$@"); break ;;
|
|
-*) echo "FATAL: unknown option '$1' (usage: $0 [--seed FILE] [--refs FILE] <zImage> <script> [out-dir])" >&2; exit 1 ;;
|
|
*) ARGS+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
ZIMAGE="${ARGS[0]:-}"
|
|
SCRIPT="${ARGS[1]:-}"
|
|
OUTDIR="${ARGS[2]:-$QDIR/out/ui-drive}"
|
|
|
|
if [ -z "$ZIMAGE" ] || [ ! -f "$ZIMAGE" ]; then
|
|
echo "FATAL: usage: $0 [--seed FILE] [--refs FILE] <zImage> <script> [out-dir]: the virt.fragment kernel variant" >&2
|
|
exit 1
|
|
fi
|
|
if [ -z "$SCRIPT" ] || [ ! -f "$SCRIPT" ]; then
|
|
echo "FATAL: no drive script at '$SCRIPT'" >&2
|
|
exit 1
|
|
fi
|
|
if [ -n "$SEED_FILE" ] && [ ! -f "$SEED_FILE" ]; then
|
|
echo "FATAL: no seed manifest at '$SEED_FILE'" >&2
|
|
exit 1
|
|
fi
|
|
if [ -n "$SEED_FILE" ] && { [ -z "${FLARE_EDGE:-}" ] || [ ! -f "$FLARE_EDGE/tools/seed-fixtures.py" ]; }; then
|
|
echo "FATAL: --seed needs FLARE_EDGE to point at a flare-edge checkout (seed-fixtures.py not found under '${FLARE_EDGE:-}')" >&2
|
|
exit 1
|
|
fi
|
|
[ -x "$QDIR/payload/warden-ui" ] || {
|
|
echo "FATAL: no qemu/payload/warden-ui: build it with flare-edge tools/build-ui-vm.sh" >&2
|
|
exit 1
|
|
}
|
|
command -v qemu-system-arm >/dev/null || {
|
|
echo "FATAL: qemu-system-arm not on PATH: see qemu/README.md" >&2
|
|
exit 1
|
|
}
|
|
|
|
mkdir -p "$OUTDIR"
|
|
# Short-named scratch: AF_UNIX socket paths are capped at ~108 chars.
|
|
WORK="$(mktemp -d /tmp/wqd.XXXXXX)"
|
|
QEMU_PID=""
|
|
cleanup() {
|
|
if [ -n "$QEMU_PID" ]; then
|
|
python3 "$HERE/qmp.py" "$WORK/qmp.sock" quit 2>/dev/null || true
|
|
sleep 1
|
|
kill "$QEMU_PID" 2>/dev/null || true
|
|
fi
|
|
cp "$WORK/console.log" "$OUTDIR/console.log" 2>/dev/null || true
|
|
rm -rf "$WORK"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
bash "$QDIR/mkinitramfs.sh"
|
|
if [ -n "$SEED_FILE" ]; then
|
|
# Stage the settings files BEFORE mkimage.sh builds the disk: userdata is
|
|
# baked into the image up front (see qemu/README.md's boundary table --
|
|
# there is no mount-after-boot step this rig could inject files through),
|
|
# so anything a screen reads at startup has to be in place before -kernel
|
|
# even runs, not written into a running VM.
|
|
python3 "$FLARE_EDGE/tools/seed-fixtures.py" "$SEED_FILE" "$WORK/seed"
|
|
SEED_DIR="$WORK/seed" bash "$QDIR/mkimage.sh"
|
|
else
|
|
bash "$QDIR/mkimage.sh"
|
|
fi
|
|
|
|
# Random hostfwd ports can collide. Detect qemu's early bind failure and retry
|
|
# with a fresh base rather than failing spuriously.
|
|
for _attempt in 1 2 3; do
|
|
PORT=$((21000 + RANDOM % 20000))
|
|
: > "$WORK/console.log"
|
|
bash "$QDIR/run.sh" --kernel "$ZIMAGE" --display headless --qmp "$WORK/qmp.sock" \
|
|
--ctl "$WORK/ctl.sock" \
|
|
--ssh-port "$PORT" --http-port $((PORT + 1)) --api-port $((PORT + 2)) \
|
|
> "$WORK/console.log" 2>&1 &
|
|
QEMU_PID=$!
|
|
sleep 3
|
|
kill -0 "$QEMU_PID" 2>/dev/null && break
|
|
if grep -aq 'Could not set up host forwarding' "$WORK/console.log"; then
|
|
echo "== hostfwd port collision on base $PORT, retrying"
|
|
QEMU_PID=""
|
|
continue
|
|
fi
|
|
echo "FATAL: VM died at launch:" >&2
|
|
tail -20 "$WORK/console.log" >&2
|
|
exit 1
|
|
done
|
|
if [ -z "$QEMU_PID" ] || ! kill -0 "$QEMU_PID" 2>/dev/null; then
|
|
echo "FATAL: could not launch the VM after 3 port attempts" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "== waiting for warden-ui"
|
|
deadline=$((SECONDS + 180))
|
|
while [ $SECONDS -lt $deadline ]; do
|
|
grep -aq 'init: starting warden-ui' "$WORK/console.log" && break
|
|
kill -0 "$QEMU_PID" 2>/dev/null || { echo "FATAL: VM exited early" >&2; tail -25 "$WORK/console.log" >&2; exit 1; }
|
|
sleep 2
|
|
done
|
|
grep -aq 'init: starting warden-ui' "$WORK/console.log" || {
|
|
echo "FATAL: warden-ui never started (no fb0? wrong kernel?)" >&2
|
|
tail -25 "$WORK/console.log" >&2
|
|
exit 1
|
|
}
|
|
|
|
# A started process is not a rendered frame. Poll screendumps until the panel
|
|
# stops being a single flat colour, on a bounded deadline: TCG renders CPU-bound
|
|
# and a loaded host can be arbitrarily slow, so this is never a fixed sleep.
|
|
echo "== waiting for the first real frame"
|
|
deadline=$((SECONDS + 180))
|
|
ready=0
|
|
while [ $SECONDS -lt $deadline ]; do
|
|
python3 "$HERE/qmp.py" "$WORK/qmp.sock" screendump "$WORK/probe.ppm" 2>/dev/null || { sleep 2; continue; }
|
|
colors="$(python3 -c "
|
|
import sys
|
|
d=open('$WORK/probe.ppm','rb').read()
|
|
print(len(set(d[i:i+3] for i in range(15, len(d), 3))))
|
|
" 2>/dev/null || echo 0)"
|
|
[ "${colors:-0}" -gt 32 ] && { ready=1; break; }
|
|
sleep 3
|
|
done
|
|
[ "$ready" = 1 ] || {
|
|
echo "FATAL: the UI never rendered a real frame" >&2
|
|
tail -25 "$WORK/console.log" >&2
|
|
exit 1
|
|
}
|
|
|
|
# The control bridge (init -> warden-ui's debug FIFO, see run.sh --ctl) comes
|
|
# up with the UI; a script's first `page`/`hit` must not race it. Bounded, and
|
|
# fail-closed: a scenario that asserts on UI state needs the channel, and a
|
|
# silently absent one would turn every assertion into an infrastructure error
|
|
# dressed as a test result.
|
|
echo "== waiting for the control bridge"
|
|
deadline=$((SECONDS + 60))
|
|
until grep -aq 'init: control bridge on' "$WORK/console.log"; do
|
|
[ $SECONDS -lt $deadline ] || {
|
|
echo "FATAL: the control bridge never announced itself (run.sh --ctl / init marker)" >&2
|
|
tail -25 "$WORK/console.log" >&2
|
|
exit 1
|
|
}
|
|
sleep 1
|
|
done
|
|
|
|
echo "== driving $SCRIPT"
|
|
# Every step lands in results.jsonl (ok / fail / fatal); an assertion mismatch
|
|
# is a `fail` and the run continues, so one run reports every broken
|
|
# expectation. The driver's exit status is the verdict; capture it rather than
|
|
# let `set -e` skip the backstop and the summary below.
|
|
drive_rc=0
|
|
DRIVE_ARGS=(--ctl "$WORK/ctl.sock" --console "$WORK/console.log")
|
|
# Only passed when given: qmp.py's own default (<outdir>/refs.json, see its
|
|
# docstring) is right for the common case of one refs file living next to a
|
|
# flow's other fixtures, and forcing a path here would just duplicate that
|
|
# default in two places.
|
|
[ -n "$REFS_FILE" ] && DRIVE_ARGS+=(--refs "$REFS_FILE")
|
|
python3 "$HERE/qmp.py" "$WORK/qmp.sock" drive "$SCRIPT" "$OUTDIR" \
|
|
"${DRIVE_ARGS[@]}" || drive_rc=$?
|
|
|
|
# The UI must still be alive: see the header. The driver checks this after
|
|
# every step and pins a crash to the step that caused it; this is the
|
|
# backstop for a death after the last step, or a driver that itself fell over.
|
|
if grep -aq 'warden-ui EXITED' "$WORK/console.log"; then
|
|
echo "FATAL: warden-ui DIED during the run:" >&2
|
|
grep -a -A22 'warden-ui EXITED' "$WORK/console.log" >&2
|
|
exit 1
|
|
fi
|
|
if [ "$drive_rc" -ne 0 ]; then
|
|
echo "UI-DRIVE-FAIL: see $OUTDIR/results.jsonl" >&2
|
|
grep -E '"status": "(fail|fatal)"' "$OUTDIR/results.jsonl" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
echo "UI-DRIVE-PASS (results in $OUTDIR/results.jsonl, screenshots in $OUTDIR)"
|