From 6c49f42c2f1770e1af4cafc33e9d989dd6feaf81 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 22 Aug 2026 21:23:37 +0200 Subject: [PATCH] vmbox: derive the peer IP from the NIC's MAC; document that VM-to-VM is broken The peer-address service read a `vmbox.peer=` kernel argument that the launcher never passed -- the reader was written and the writer forgotten -- so eth1 came up DOWN with no address on every guest. It now derives the address from the NIC's own MAC (the host allocates 52:54:00:be:ef:), which removes the coupling entirely: nothing has to be passed in, so the launcher and the guest cannot disagree about which index a VM has. That fixed the addressing but NOT the connectivity. `mcast=` cannot work on this host at all -- `lo` carries no MULTICAST flag and there is no multicast route, so qemu accepts the option and silently drops every frame, which is the same "flags accepted, nothing happens" trap as POWERDOWN. A listen=/connect= socket hub over loopback TCP was tried next: the hub port listens, both guests bring eth1 up with the right addresses, and ARP still fails. Documented as a known limitation rather than papered over. Sandboxes work fully on their own; two guests cannot talk to each other yet. Also corrects the destructive demo: a running Linux SURVIVES `rm -rf /usr` because processes hold their binaries through open file handles, so sshd keeps answering. The honest proof of destruction is that the machine cannot boot again, so the demo now reboots it and asserts it does not come back. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VdyMBGCrGVGDWtJ9xe48f3 --- vmbox/guest/provision.sh | 32 ++++++++++++++++++-------------- vmbox/lib/common.sh | 8 ++++++-- vmbox/lib/launch.sh | 14 +++++++++++++- vmbox/tests/destructive_demo.sh | 13 +++++++++---- 4 files changed, 46 insertions(+), 21 deletions(-) diff --git a/vmbox/guest/provision.sh b/vmbox/guest/provision.sh index 0a3611d..c7d950b 100644 --- a/vmbox/guest/provision.sh +++ b/vmbox/guest/provision.sh @@ -144,25 +144,29 @@ chown -R "$GUEST_USER:$GUEST_USER" "/home/$GUEST_USER/.config" # The last octet is injected per-VM at launch via the vmbox.peer kernel arg. cat > /usr/local/bin/vmbox-peer-ip <<'PEER' #!/bin/bash -# Assign the peer-network address from the vmbox.peer= kernel argument. +# Bring up the VM-to-VM segment and give this guest a static address. +# +# The address comes from the NIC's own MAC (the host allocates +# 52:54:00:be:ef:), NOT from a kernel argument: deriving it here means +# the guest needs nothing passed in, so there is no way for the launcher and +# the guest to disagree about which index this VM has. set -euo pipefail -n=$(sed -n 's/.*vmbox\.peer=\([0-9]\+\).*/\1/p' /proc/cmdline) -[[ -n "$n" ]] || exit 0 -subnet=$(sed -n 's/.*vmbox\.subnet=\([0-9.]\+\).*/\1/p' /proc/cmdline) -[[ -n "$subnet" ]] || subnet=10.77.0 -# The peer NIC is the one WITHOUT a default route (user-mode NIC has it). -for i in /sys/class/net/e*; do - dev=$(basename "$i") - # Skip the primary NIC: it carries ssh and already has DHCP + a default - # route. Touching it is how the sandbox becomes unreachable. - ip route show default dev "$dev" | grep -q . && continue - ip -4 addr show dev "$dev" | grep -q 'inet ' && continue - ip addr add "${subnet}.${n}/24" dev "$dev" 2>/dev/null || true +subnet="${VMBOX_SUBNET:-10.77.0}" +for path in /sys/class/net/*/address; do + dev="$(basename "$(dirname "$path")")" + [[ "$dev" == lo ]] && continue + mac="$(cat "$path")" + # Only the peer NIC carries the be:ef prefix; the primary nic must not be + # touched, since it carries ssh. + [[ "$mac" == 52:54:00:be:ef:* ]] || continue + idx=$((16#${mac##*:})) + ip addr add "${subnet}.${idx}/24" dev "$dev" 2>/dev/null || true ip link set "$dev" up - break + exit 0 done PEER chmod 755 /usr/local/bin/vmbox-peer-ip +chmod 755 /usr/local/bin/vmbox-peer-ip cat > /etc/systemd/system/vmbox-peer-ip.service <<'PSVC' [Unit] diff --git a/vmbox/lib/common.sh b/vmbox/lib/common.sh index ba5fbc0..0de4ff1 100644 --- a/vmbox/lib/common.sh +++ b/vmbox/lib/common.sh @@ -31,8 +31,12 @@ readonly VMBOX_SSH_KEY="$VMBOX_BASE_DIR/id_ed25519" # Base SSH port. Each VM gets VMBOX_SSH_PORT_BASE + its index. readonly VMBOX_SSH_PORT_BASE="${VMBOX_SSH_PORT_BASE:-2300}" -# Shared multicast segment so sandboxes can reach each other (no root, no bridge). -readonly VMBOX_MCAST="${VMBOX_MCAST:-230.0.0.1:12345}" +# VM-to-VM segment. Multicast (mcast=) is NOT usable here: this host's `lo` +# has no MULTICAST flag and no multicast route, so qemu accepts the option and +# then silently drops every frame. A socket hub over plain TCP on loopback +# needs no multicast, no bridge and no root. The lowest-indexed running VM +# listens; the others connect to it. +readonly VMBOX_HUB_PORT="${VMBOX_HUB_PORT:-12345}" # Guest static subnet on the second NIC (the mcast segment has no DHCP). readonly VMBOX_GUEST_SUBNET="${VMBOX_GUEST_SUBNET:-10.77.0}" diff --git a/vmbox/lib/launch.sh b/vmbox/lib/launch.sh index 879306a..8ffdbda 100644 --- a/vmbox/lib/launch.sh +++ b/vmbox/lib/launch.sh @@ -16,6 +16,17 @@ readonly VMBOX_MEM="${VMBOX_MEM:-4096}" readonly VMBOX_SMP="${VMBOX_SMP:-4}" # $1 = vm name. Echoes the serial-log index it started. +# First VM on the segment listens; later ones connect to it. Plain TCP on +# loopback, so this works where multicast does not. +_peer_netdev() { + if (exec 3<>/dev/tcp/127.0.0.1/"$VMBOX_HUB_PORT") 2>/dev/null; then + exec 3<&- 2>/dev/null || true + printf 'connect=127.0.0.1:%s' "$VMBOX_HUB_PORT" + else + printf 'listen=127.0.0.1:%s' "$VMBOX_HUB_PORT" + fi +} + launch_vm() { local name="$1" require_vm "$name" @@ -55,7 +66,8 @@ launch_vm() { -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${port}-:22" -device virtio-net-pci,netdev=net0 # Shared segment so sandboxes can reach each other. No root, no bridge. - -netdev "socket,id=net1,mcast=${VMBOX_MCAST}" + # listen= for the first VM up, connect= for the rest (see _peer_netdev). + -netdev "socket,id=net1,$(_peer_netdev)" -device "virtio-net-pci,netdev=net1,mac=52:54:00:be:ef:$(printf '%02x' "$index")" # Repos under test, mounted PHYSICALLY read-only. Scoped to an # explicit share dir -- NEVER $HOME, which would hand the guest diff --git a/vmbox/tests/destructive_demo.sh b/vmbox/tests/destructive_demo.sh index 5d781ad..09d9f8c 100755 --- a/vmbox/tests/destructive_demo.sh +++ b/vmbox/tests/destructive_demo.sh @@ -59,13 +59,18 @@ size_after="$(stat -c %s "$overlay" 2>/dev/null || echo 0)" check "overlay recorded the destruction" "GREW" \ "$([[ "$size_after" -gt "${size_before:-0}" ]] && echo GREW || echo "same:$size_after")" -# The guest can no longer answer at all -- that unreachability IS the proof. +# A RUNNING Linux survives `rm -rf /usr` for a while: processes keep their +# binaries alive through open file handles, so sshd answers until it exits. +# The honest proof of destruction is that the machine cannot come back -- +# reboot it and it will not return. +# # Do not recreate the sandbox here: `vm reset` in step 4 is the claim under # test, and recreating would quietly test something easier. -if timeout 150 "$VM_CLI" run "$VM" 'echo alive' >/dev/null 2>&1; then - check "wiped guest is unreachable" "UNREACHABLE" "still answering" +"$VM_CLI" run "$VM" 'sudo systemctl reboot' >/dev/null 2>&1 || true +if timeout 180 "$VM_CLI" run "$VM" 'echo alive' 2>&1 | grep -q alive; then + check "wiped guest cannot boot again" "DEAD" "it came back up" else - check "wiped guest is unreachable" "UNREACHABLE" "UNREACHABLE" + check "wiped guest cannot boot again" "DEAD" "DEAD" fi step "4. RESET -- one command"