diff --git a/vmbox/README.md b/vmbox/README.md index 10caafc..ecde8d4 100644 --- a/vmbox/README.md +++ b/vmbox/README.md @@ -179,7 +179,7 @@ stay on the host or stay untested. ## Tests -`bats tests/test_vmbox.bats tests/test_rtcwake.bats` — host-side unit tests +`bats tests/` — host-side unit tests for name validation, meta handling, serial rotation, the full verdict table, firmware selection and the CMOS alarm decoder. They do not boot a VM; booting is covered by the end-to-end demos (`tests/*_demo.sh`, `tests/rtcwake_e2e.sh`). diff --git a/vmbox/tests/test_helper.bash b/vmbox/tests/test_helper.bash new file mode 100644 index 0000000..3bb5e66 --- /dev/null +++ b/vmbox/tests/test_helper.bash @@ -0,0 +1,30 @@ +# ============================================================================ +# vmbox: shared bats fixture for the host-side unit tests (load test_helper). +# +# VMBOX_HOME is redirected to a per-test temp dir. Nothing here may touch the +# real ~/.local/share/vmbox: these tests create and delete VM state, and a +# stray write there would clobber a live sandbox's overlay or pidfile. +# ============================================================================ + +setup() { + VMBOX_TEST_HOME="$(mktemp -d)" + export VMBOX_HOME="$VMBOX_TEST_HOME" + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd -P)" + export REPO_ROOT + + # common.sh guards against double-sourcing with a readonly flag, and bats + # runs every test in a fresh subshell, so a plain source is safe here. + source "$REPO_ROOT/lib/common.sh" + source "$REPO_ROOT/lib/verdict.sh" + + install -d "$VMBOX_VMS_DIR" +} + +teardown() { + [[ -n "${VMBOX_TEST_HOME:-}" && -d "$VMBOX_TEST_HOME" ]] && rm -rf "$VMBOX_TEST_HOME" +} + +# Create a fake VM directory without booting anything. +_fake_vm() { + install -d "$(vm_dir "$1")" +} diff --git a/vmbox/tests/test_shim.bats b/vmbox/tests/test_shim.bats new file mode 100644 index 0000000..645c96d --- /dev/null +++ b/vmbox/tests/test_shim.bats @@ -0,0 +1,44 @@ +#!/usr/bin/env bats +# ============================================================================ +# vmbox: host-side unit tests for guest/shim.sh, the fake hardware tools. +# +# The shim runs on the host against a temp PATH under the per-test VMBOX_HOME +# from test_helper.bash; nothing is installed system-wide. +# ============================================================================ + +load test_helper + +_shim() { + VMBOX_SHIM_DIR="$VMBOX_TEST_HOME/sbin" VMBOX_SHIM_LOG_DIR="$VMBOX_TEST_HOME/shimlog" \ + bash "$REPO_ROOT/guest/shim.sh" "$@" +} + +@test "shim: exit mode returns 0 and logs the argv with an epoch" { + run _shim openrgb exit + [ "$status" -eq 0 ] + run "$VMBOX_TEST_HOME/sbin/openrgb" --mode static --color 000000 + [ "$status" -eq 0 ] + run cat "$VMBOX_TEST_HOME/shimlog/openrgb.log" + [[ "$output" =~ ^[0-9]{9,}\ --mode\ static\ --color\ 000000$ ]] +} + +@test "shim: exit:N returns N" { + _shim nvidia-smi exit:3 + run "$VMBOX_TEST_HOME/sbin/nvidia-smi" -pl 100 + [ "$status" -eq 3 ] +} + +@test "shim: hang mode never returns (the 2026-09-12 openrgb)" { + _shim openrgb hang + run timeout 1 "$VMBOX_TEST_HOME/sbin/openrgb" + [ "$status" -eq 124 ] + run cat "$VMBOX_TEST_HOME/shimlog/openrgb.log" + [ "${#lines[@]}" -eq 1 ] +} + +@test "shim: an unknown mode or a path is refused" { + run _shim openrgb explode + [ "$status" -eq 1 ] + run _shim /usr/bin/openrgb exit + [ "$status" -eq 1 ] +} diff --git a/vmbox/tests/test_verdict.bats b/vmbox/tests/test_verdict.bats new file mode 100644 index 0000000..381f9c6 --- /dev/null +++ b/vmbox/tests/test_verdict.bats @@ -0,0 +1,182 @@ +#!/usr/bin/env bats +# ============================================================================ +# vmbox: host-side unit tests for the shutdown verdict. +# +# Event parsing, serial completeness and the full exit-code table, all decided +# from fake event logs and serial text -- no guest is booted. Fixture (temp +# VMBOX_HOME, _fake_vm) is test_helper.bash. +# ============================================================================ + +load test_helper + +# --------------------------------------------------------------------------- +# verdict: event parsing +# --------------------------------------------------------------------------- + +@test "verdict_last_event reports none for a missing or empty log" { + run verdict_last_event "$VMBOX_HOME/nosuch.jsonl" + [ "$output" = "none" ] + : > "$VMBOX_HOME/empty.jsonl" + run verdict_last_event "$VMBOX_HOME/empty.jsonl" + [ "$output" = "none" ] +} + +@test "verdict_last_event reads a guest-initiated poweroff" { + printf '%s\n' \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ + > "$VMBOX_HOME/e.jsonl" + run verdict_last_event "$VMBOX_HOME/e.jsonl" + [ "$output" = "guest-shutdown" ] +} + +@test "verdict_last_event distinguishes a reset from a poweroff" { + printf '%s\n' \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-reset"}}' \ + > "$VMBOX_HOME/e.jsonl" + run verdict_last_event "$VMBOX_HOME/e.jsonl" + [ "$output" = "guest-reset" ] +} + +@test "verdict_last_event reports a panic" { + printf '%s\n' '{"event":"GUEST_PANICKED","data":{"action":"pause"}}' \ + > "$VMBOX_HOME/e.jsonl" + run verdict_last_event "$VMBOX_HOME/e.jsonl" + [ "$output" = "panic" ] +} + +@test "verdict_last_event takes the LAST event, not the first" { + # events.jsonl is append-only across boots, so an old poweroff must never + # be mistaken for the current run's outcome. + printf '%s\n' \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-reset"}}' \ + > "$VMBOX_HOME/e.jsonl" + run verdict_last_event "$VMBOX_HOME/e.jsonl" + [ "$output" = "guest-reset" ] +} + +@test "verdict_last_event skips malformed lines instead of dying" { + # A truncated final line is normal: the recorder is killed with the VM. + printf '%s\n' \ + 'not json at all' \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ + '{"event":"SHUT' \ + > "$VMBOX_HOME/e.jsonl" + run verdict_last_event "$VMBOX_HOME/e.jsonl" + [ "$status" -eq 0 ] + [ "$output" = "guest-shutdown" ] +} + +@test "verdict_last_event ignores POWERDOWN, which proves nothing" { + # POWERDOWN means the ACPI request was DELIVERED. With no OS booted, qemu + # emits it and then runs forever -- treating it as a stop is a false pass. + printf '%s\n' '{"event":"POWERDOWN"}' > "$VMBOX_HOME/e.jsonl" + run verdict_last_event "$VMBOX_HOME/e.jsonl" + [ "$output" = "none" ] +} + +@test "verdict_last_event marks a host-initiated stop as host" { + printf '%s\n' '{"event":"SHUTDOWN","data":{"guest":false}}' \ + > "$VMBOX_HOME/e.jsonl" + run verdict_last_event "$VMBOX_HOME/e.jsonl" + [ "$output" = "host" ] +} + +# --------------------------------------------------------------------------- +# verdict: serial completeness +# --------------------------------------------------------------------------- + +@test "verdict_serial_complete requires the kernel's final power-down line" { + printf 'Reached target System Power Off.\n' > "$VMBOX_HOME/s.log" + run verdict_serial_complete "$VMBOX_HOME/s.log" + [ "$status" -ne 0 ] + + printf 'Reached target System Power Off.\n[ 85.5] reboot: Power down\n' \ + > "$VMBOX_HOME/s.log" + run verdict_serial_complete "$VMBOX_HOME/s.log" + [ "$status" -eq 0 ] +} + +@test "verdict_serial_complete fails on an empty or missing log" { + : > "$VMBOX_HOME/s.log" + run verdict_serial_complete "$VMBOX_HOME/s.log" + [ "$status" -ne 0 ] + run verdict_serial_complete "$VMBOX_HOME/nosuch.log" + [ "$status" -ne 0 ] +} + +@test "verdict_serial_complete matches despite ANSI escapes in the log" { + # The serial log is a raw console capture, full of colour codes; grep -a + # is what keeps it readable as text. + printf '\033[0;32m OK \033[0m[ 85.5] reboot: Power down\n' \ + > "$VMBOX_HOME/s.log" + run verdict_serial_complete "$VMBOX_HOME/s.log" + [ "$status" -eq 0 ] +} + +# --------------------------------------------------------------------------- +# verdict: the full table, exit code by exit code +# --------------------------------------------------------------------------- + +_verdict_case() { + # _verdict_case -> sets $status + local name="v1" + _fake_vm "$name" + local events serial + events="$(vm_events "$name")" + serial="$(vm_serial "$name" 0)" + if [[ -n "$1" ]]; then printf '%s\n' "$1" > "$events"; else : > "$events"; fi + printf '%s' "$2" > "$serial" + verdict_report "$name" "$serial" "$events" +} + +@test "verdict: clean poweroff exits 0" { + run _verdict_case \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ + 'reboot: Power down' + [ "$status" -eq 0 ] + [[ "$output" == *"clean poweroff"* ]] +} + +@test "verdict: a stop that never reached the marker is DIRTY, exit 3" { + # This is the discriminator the whole design turns on: guest:true alone + # does NOT prove the machine shut down completely. + run _verdict_case \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ + 'Stopping session... (log ends here)' + [ "$status" -eq 3 ] + [[ "$output" == *"DIRTY"* ]] +} + +@test "verdict: a reboot is exit 4, not a poweroff" { + run _verdict_case \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-reset"}}' \ + '' + [ "$status" -eq 4 ] + [[ "$output" == *"REBOOTED"* ]] +} + +@test "verdict: a kernel panic is exit 5" { + run _verdict_case '{"event":"GUEST_PANICKED","data":{"action":"pause"}}' '' + [ "$status" -eq 5 ] + [[ "$output" == *"PANIC"* ]] +} + +@test "verdict: no event and no live qemu is exit 6" { + # No pidfile -> vm_is_running is false -> stopped without a guest event. + run _verdict_case '' '' + [ "$status" -eq 6 ] +} + +@test "verdict: a host-initiated stop is exit 6" { + run _verdict_case '{"event":"SHUTDOWN","data":{"guest":false}}' '' + [ "$status" -eq 6 ] +} + +@test "verdict: an unrecognised reason is exit 6, never a silent pass" { + run _verdict_case \ + '{"event":"SHUTDOWN","data":{"guest":true,"reason":"brand-new-reason"}}' \ + '' + [ "$status" -eq 6 ] + [[ "$output" == *"unrecognised"* ]] +} diff --git a/vmbox/tests/test_vmbox.bats b/vmbox/tests/test_vmbox.bats index fab1840..f1afc5d 100644 --- a/vmbox/tests/test_vmbox.bats +++ b/vmbox/tests/test_vmbox.bats @@ -3,37 +3,15 @@ # vmbox: host-side unit tests. # # Scope, deliberately: everything that can be decided WITHOUT booting a guest -- -# name validation, meta handling, serial rotation and the full verdict table. +# name validation, meta handling, serial rotation and the VM path helpers. # Booting is covered by tests/destructive_demo.sh, which is slow and needs a # built base image; these run in well under a second and gate every commit. # -# VMBOX_HOME is redirected to a per-test temp dir. Nothing here may touch the -# real ~/.local/share/vmbox: these tests create and delete VM state, and a -# stray write there would clobber a live sandbox's overlay or pidfile. +# The verdict table lives in test_verdict.bats and the hardware shim in +# test_shim.bats; the shared fixture (temp VMBOX_HOME) is test_helper.bash. # ============================================================================ -setup() { - VMBOX_TEST_HOME="$(mktemp -d)" - export VMBOX_HOME="$VMBOX_TEST_HOME" - REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd -P)" - export REPO_ROOT - - # common.sh guards against double-sourcing with a readonly flag, and bats - # runs every test in a fresh subshell, so a plain source is safe here. - source "$REPO_ROOT/lib/common.sh" - source "$REPO_ROOT/lib/verdict.sh" - - install -d "$VMBOX_VMS_DIR" -} - -teardown() { - [[ -n "${VMBOX_TEST_HOME:-}" && -d "$VMBOX_TEST_HOME" ]] && rm -rf "$VMBOX_TEST_HOME" -} - -# Create a fake VM directory without booting anything. -_fake_vm() { - install -d "$(vm_dir "$1")" -} +load test_helper # --------------------------------------------------------------------------- # name validation @@ -217,213 +195,3 @@ _fake_vm() { run require_vm yes1 [ "$status" -eq 0 ] } - -# --------------------------------------------------------------------------- -# verdict: event parsing -# --------------------------------------------------------------------------- - -@test "verdict_last_event reports none for a missing or empty log" { - run verdict_last_event "$VMBOX_HOME/nosuch.jsonl" - [ "$output" = "none" ] - : > "$VMBOX_HOME/empty.jsonl" - run verdict_last_event "$VMBOX_HOME/empty.jsonl" - [ "$output" = "none" ] -} - -@test "verdict_last_event reads a guest-initiated poweroff" { - printf '%s\n' \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ - > "$VMBOX_HOME/e.jsonl" - run verdict_last_event "$VMBOX_HOME/e.jsonl" - [ "$output" = "guest-shutdown" ] -} - -@test "verdict_last_event distinguishes a reset from a poweroff" { - printf '%s\n' \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-reset"}}' \ - > "$VMBOX_HOME/e.jsonl" - run verdict_last_event "$VMBOX_HOME/e.jsonl" - [ "$output" = "guest-reset" ] -} - -@test "verdict_last_event reports a panic" { - printf '%s\n' '{"event":"GUEST_PANICKED","data":{"action":"pause"}}' \ - > "$VMBOX_HOME/e.jsonl" - run verdict_last_event "$VMBOX_HOME/e.jsonl" - [ "$output" = "panic" ] -} - -@test "verdict_last_event takes the LAST event, not the first" { - # events.jsonl is append-only across boots, so an old poweroff must never - # be mistaken for the current run's outcome. - printf '%s\n' \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-reset"}}' \ - > "$VMBOX_HOME/e.jsonl" - run verdict_last_event "$VMBOX_HOME/e.jsonl" - [ "$output" = "guest-reset" ] -} - -@test "verdict_last_event skips malformed lines instead of dying" { - # A truncated final line is normal: the recorder is killed with the VM. - printf '%s\n' \ - 'not json at all' \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ - '{"event":"SHUT' \ - > "$VMBOX_HOME/e.jsonl" - run verdict_last_event "$VMBOX_HOME/e.jsonl" - [ "$status" -eq 0 ] - [ "$output" = "guest-shutdown" ] -} - -@test "verdict_last_event ignores POWERDOWN, which proves nothing" { - # POWERDOWN means the ACPI request was DELIVERED. With no OS booted, qemu - # emits it and then runs forever -- treating it as a stop is a false pass. - printf '%s\n' '{"event":"POWERDOWN"}' > "$VMBOX_HOME/e.jsonl" - run verdict_last_event "$VMBOX_HOME/e.jsonl" - [ "$output" = "none" ] -} - -@test "verdict_last_event marks a host-initiated stop as host" { - printf '%s\n' '{"event":"SHUTDOWN","data":{"guest":false}}' \ - > "$VMBOX_HOME/e.jsonl" - run verdict_last_event "$VMBOX_HOME/e.jsonl" - [ "$output" = "host" ] -} - -# --------------------------------------------------------------------------- -# verdict: serial completeness -# --------------------------------------------------------------------------- - -@test "verdict_serial_complete requires the kernel's final power-down line" { - printf 'Reached target System Power Off.\n' > "$VMBOX_HOME/s.log" - run verdict_serial_complete "$VMBOX_HOME/s.log" - [ "$status" -ne 0 ] - - printf 'Reached target System Power Off.\n[ 85.5] reboot: Power down\n' \ - > "$VMBOX_HOME/s.log" - run verdict_serial_complete "$VMBOX_HOME/s.log" - [ "$status" -eq 0 ] -} - -@test "verdict_serial_complete fails on an empty or missing log" { - : > "$VMBOX_HOME/s.log" - run verdict_serial_complete "$VMBOX_HOME/s.log" - [ "$status" -ne 0 ] - run verdict_serial_complete "$VMBOX_HOME/nosuch.log" - [ "$status" -ne 0 ] -} - -@test "verdict_serial_complete matches despite ANSI escapes in the log" { - # The serial log is a raw console capture, full of colour codes; grep -a - # is what keeps it readable as text. - printf '\033[0;32m OK \033[0m[ 85.5] reboot: Power down\n' \ - > "$VMBOX_HOME/s.log" - run verdict_serial_complete "$VMBOX_HOME/s.log" - [ "$status" -eq 0 ] -} - -# --------------------------------------------------------------------------- -# verdict: the full table, exit code by exit code -# --------------------------------------------------------------------------- - -_verdict_case() { - # _verdict_case -> sets $status - local name="v1" - _fake_vm "$name" - local events serial - events="$(vm_events "$name")" - serial="$(vm_serial "$name" 0)" - if [[ -n "$1" ]]; then printf '%s\n' "$1" > "$events"; else : > "$events"; fi - printf '%s' "$2" > "$serial" - verdict_report "$name" "$serial" "$events" -} - -@test "verdict: clean poweroff exits 0" { - run _verdict_case \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ - 'reboot: Power down' - [ "$status" -eq 0 ] - [[ "$output" == *"clean poweroff"* ]] -} - -@test "verdict: a stop that never reached the marker is DIRTY, exit 3" { - # This is the discriminator the whole design turns on: guest:true alone - # does NOT prove the machine shut down completely. - run _verdict_case \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-shutdown"}}' \ - 'Stopping session... (log ends here)' - [ "$status" -eq 3 ] - [[ "$output" == *"DIRTY"* ]] -} - -@test "verdict: a reboot is exit 4, not a poweroff" { - run _verdict_case \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"guest-reset"}}' \ - '' - [ "$status" -eq 4 ] - [[ "$output" == *"REBOOTED"* ]] -} - -@test "verdict: a kernel panic is exit 5" { - run _verdict_case '{"event":"GUEST_PANICKED","data":{"action":"pause"}}' '' - [ "$status" -eq 5 ] - [[ "$output" == *"PANIC"* ]] -} - -@test "verdict: no event and no live qemu is exit 6" { - # No pidfile -> vm_is_running is false -> stopped without a guest event. - run _verdict_case '' '' - [ "$status" -eq 6 ] -} - -@test "verdict: a host-initiated stop is exit 6" { - run _verdict_case '{"event":"SHUTDOWN","data":{"guest":false}}' '' - [ "$status" -eq 6 ] -} - -@test "verdict: an unrecognised reason is exit 6, never a silent pass" { - run _verdict_case \ - '{"event":"SHUTDOWN","data":{"guest":true,"reason":"brand-new-reason"}}' \ - '' - [ "$status" -eq 6 ] - [[ "$output" == *"unrecognised"* ]] -} - -# --------------------------------------------------------------------------- -# guest/shim.sh -- fake hardware tools (runs on the host against a temp PATH) -# --------------------------------------------------------------------------- -_shim() { - VMBOX_SHIM_DIR="$VMBOX_TEST_HOME/sbin" VMBOX_SHIM_LOG_DIR="$VMBOX_TEST_HOME/shimlog" \ - bash "$REPO_ROOT/guest/shim.sh" "$@" -} - -@test "shim: exit mode returns 0 and logs the argv with an epoch" { - run _shim openrgb exit - [ "$status" -eq 0 ] - run "$VMBOX_TEST_HOME/sbin/openrgb" --mode static --color 000000 - [ "$status" -eq 0 ] - run cat "$VMBOX_TEST_HOME/shimlog/openrgb.log" - [[ "$output" =~ ^[0-9]{9,}\ --mode\ static\ --color\ 000000$ ]] -} - -@test "shim: exit:N returns N" { - _shim nvidia-smi exit:3 - run "$VMBOX_TEST_HOME/sbin/nvidia-smi" -pl 100 - [ "$status" -eq 3 ] -} - -@test "shim: hang mode never returns (the 2026-09-12 openrgb)" { - _shim openrgb hang - run timeout 1 "$VMBOX_TEST_HOME/sbin/openrgb" - [ "$status" -eq 124 ] - run cat "$VMBOX_TEST_HOME/shimlog/openrgb.log" - [ "${#lines[@]}" -eq 1 ] -} - -@test "shim: an unknown mode or a path is refused" { - run _shim openrgb explode - [ "$status" -eq 1 ] - run _shim /usr/bin/openrgb exit - [ "$status" -eq 1 ] -}