vmbox: derive the peer IP from the NIC's MAC; document that VM-to-VM is broken

The peer-address service read a `vmbox.peer=` kernel argument that the
launcher never passed -- the reader was written and the writer forgotten -- so
eth1 came up DOWN with no address on every guest. It now derives the address
from the NIC's own MAC (the host allocates 52:54:00:be:ef:<index>), which
removes the coupling entirely: nothing has to be passed in, so the launcher
and the guest cannot disagree about which index a VM has.

That fixed the addressing but NOT the connectivity. `mcast=` cannot work on
this host at all -- `lo` carries no MULTICAST flag and there is no multicast
route, so qemu accepts the option and silently drops every frame, which is the
same "flags accepted, nothing happens" trap as POWERDOWN. A listen=/connect=
socket hub over loopback TCP was tried next: the hub port listens, both
guests bring eth1 up with the right addresses, and ARP still fails.

Documented as a known limitation rather than papered over. Sandboxes work
fully on their own; two guests cannot talk to each other yet.

Also corrects the destructive demo: a running Linux SURVIVES `rm -rf /usr`
because processes hold their binaries through open file handles, so sshd keeps
answering. The honest proof of destruction is that the machine cannot boot
again, so the demo now reboots it and asserts it does not come back.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdyMBGCrGVGDWtJ9xe48f3

vmbox/guest/provision.sh

@@ -144,25 +144,29 @@ chown -R "$GUEST_USER:$GUEST_USER" "/home/$GUEST_USER/.config"
 # The last octet is injected per-VM at launch via the vmbox.peer kernel arg.
 cat > /usr/local/bin/vmbox-peer-ip <<'PEER'
 #!/bin/bash
-# Assign the peer-network address from the vmbox.peer=<n> kernel argument.
+# Bring up the VM-to-VM segment and give this guest a static address.
+#
+# The address comes from the NIC's own MAC (the host allocates
+# 52:54:00:be:ef:<index>), NOT from a kernel argument: deriving it here means
+# the guest needs nothing passed in, so there is no way for the launcher and
+# the guest to disagree about which index this VM has.
 set -euo pipefail
-n=$(sed -n 's/.*vmbox\.peer=\([0-9]\+\).*/\1/p' /proc/cmdline)
-[[ -n "$n" ]] || exit 0
-subnet=$(sed -n 's/.*vmbox\.subnet=\([0-9.]\+\).*/\1/p' /proc/cmdline)
-[[ -n "$subnet" ]] || subnet=10.77.0
-# The peer NIC is the one WITHOUT a default route (user-mode NIC has it).
-for i in /sys/class/net/e*; do
-    dev=$(basename "$i")
-    # Skip the primary NIC: it carries ssh and already has DHCP + a default
-    # route. Touching it is how the sandbox becomes unreachable.
-    ip route show default dev "$dev" | grep -q . && continue
-    ip -4 addr show dev "$dev" | grep -q 'inet ' && continue
-    ip addr add "${subnet}.${n}/24" dev "$dev" 2>/dev/null || true
+subnet="${VMBOX_SUBNET:-10.77.0}"
+for path in /sys/class/net/*/address; do
+    dev="$(basename "$(dirname "$path")")"
+    [[ "$dev" == lo ]] && continue
+    mac="$(cat "$path")"
+    # Only the peer NIC carries the be:ef prefix; the primary nic must not be
+    # touched, since it carries ssh.
+    [[ "$mac" == 52:54:00:be:ef:* ]] || continue
+    idx=$((16#${mac##*:}))
+    ip addr add "${subnet}.${idx}/24" dev "$dev" 2>/dev/null || true
     ip link set "$dev" up
-    break
+    exit 0
 done
 PEER
 chmod 755 /usr/local/bin/vmbox-peer-ip
+chmod 755 /usr/local/bin/vmbox-peer-ip
 
 cat > /etc/systemd/system/vmbox-peer-ip.service <<'PSVC'
 [Unit]

vmbox/lib/common.sh

@@ -31,8 +31,12 @@ readonly VMBOX_SSH_KEY="$VMBOX_BASE_DIR/id_ed25519"
 
 # Base SSH port. Each VM gets VMBOX_SSH_PORT_BASE + its index.
 readonly VMBOX_SSH_PORT_BASE="${VMBOX_SSH_PORT_BASE:-2300}"
-# Shared multicast segment so sandboxes can reach each other (no root, no bridge).
-readonly VMBOX_MCAST="${VMBOX_MCAST:-230.0.0.1:12345}"
+# VM-to-VM segment. Multicast (mcast=) is NOT usable here: this host's `lo`
+# has no MULTICAST flag and no multicast route, so qemu accepts the option and
+# then silently drops every frame. A socket hub over plain TCP on loopback
+# needs no multicast, no bridge and no root. The lowest-indexed running VM
+# listens; the others connect to it.
+readonly VMBOX_HUB_PORT="${VMBOX_HUB_PORT:-12345}"
 # Guest static subnet on the second NIC (the mcast segment has no DHCP).
 readonly VMBOX_GUEST_SUBNET="${VMBOX_GUEST_SUBNET:-10.77.0}"
 

vmbox/lib/launch.sh

@@ -16,6 +16,17 @@ readonly VMBOX_MEM="${VMBOX_MEM:-4096}"
 readonly VMBOX_SMP="${VMBOX_SMP:-4}"
 
 # $1 = vm name. Echoes the serial-log index it started.
+# First VM on the segment listens; later ones connect to it. Plain TCP on
+# loopback, so this works where multicast does not.
+_peer_netdev() {
+    if (exec 3<>/dev/tcp/127.0.0.1/"$VMBOX_HUB_PORT") 2>/dev/null; then
+        exec 3<&- 2>/dev/null || true
+        printf 'connect=127.0.0.1:%s' "$VMBOX_HUB_PORT"
+    else
+        printf 'listen=127.0.0.1:%s' "$VMBOX_HUB_PORT"
+    fi
+}
+
 launch_vm() {
     local name="$1"
     require_vm "$name"
@@ -55,7 +66,8 @@ launch_vm() {
         -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${port}-:22"
         -device virtio-net-pci,netdev=net0
         # Shared segment so sandboxes can reach each other. No root, no bridge.
-        -netdev "socket,id=net1,mcast=${VMBOX_MCAST}"
+        # listen= for the first VM up, connect= for the rest (see _peer_netdev).
+        -netdev "socket,id=net1,$(_peer_netdev)"
         -device "virtio-net-pci,netdev=net1,mac=52:54:00:be:ef:$(printf '%02x' "$index")"
         # Repos under test, mounted PHYSICALLY read-only. Scoped to an
         # explicit share dir -- NEVER $HOME, which would hand the guest

vmbox/tests/destructive_demo.sh

@@ -59,13 +59,18 @@ size_after="$(stat -c %s "$overlay" 2>/dev/null || echo 0)"
 check "overlay recorded the destruction" "GREW" \
     "$([[ "$size_after" -gt "${size_before:-0}" ]] && echo GREW || echo "same:$size_after")"
 
-# The guest can no longer answer at all -- that unreachability IS the proof.
+# A RUNNING Linux survives `rm -rf /usr` for a while: processes keep their
+# binaries alive through open file handles, so sshd answers until it exits.
+# The honest proof of destruction is that the machine cannot come back --
+# reboot it and it will not return.
+#
 # Do not recreate the sandbox here: `vm reset` in step 4 is the claim under
 # test, and recreating would quietly test something easier.
-if timeout 150 "$VM_CLI" run "$VM" 'echo alive' >/dev/null 2>&1; then
-    check "wiped guest is unreachable" "UNREACHABLE" "still answering"
+"$VM_CLI" run "$VM" 'sudo systemctl reboot' >/dev/null 2>&1 || true
+if timeout 180 "$VM_CLI" run "$VM" 'echo alive' 2>&1 | grep -q alive; then
+    check "wiped guest cannot boot again" "DEAD" "it came back up"
 else
-    check "wiped guest is unreachable" "UNREACHABLE" "UNREACHABLE"
+    check "wiped guest cannot boot again" "DEAD" "DEAD"
 fi
 
 step "4. RESET -- one command"