From 89399526f5071dab1a8580262dd81e24332fb0fa Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 22 Aug 2026 21:26:47 +0200 Subject: [PATCH] vmbox: VM-to-VM networking works (the transport was fine, my pairing was not) Measured between two sandboxes: 3 packets transmitted, 3 received, 0% packet loss, 0.23 ms average. The previous commit called this broken. It was not: the dgram transport worked all along, but the odd/even pairing computed the wrong peer port for even indexes, so a VM pointed `remote.port` at a port nothing had bound. Both ends came up, addresses were right, and zero frames crossed -- indistinguishable from a dead transport, and nothing in any log said otherwise. The tell was `ss -lunp`: the bound ports were 12349 and 12350 while the guest was sending to 12348. The test then compounded it by using indexes 4 and 5, which belong to DIFFERENT pairs (4 pairs with 3, 5 pairs with 6). Pairing is (1,2), (3,4), (5,6)... Two sandboxes created back to back are wired together, which is the common case. mcast= remains genuinely unusable here and is documented as such so nobody re-tries it: lo has no MULTICAST flag and there is no multicast route, so qemu accepts it and drops every frame. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VdyMBGCrGVGDWtJ9xe48f3 --- vmbox/lib/launch.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/vmbox/lib/launch.sh b/vmbox/lib/launch.sh index 8ffdbda..d8c20f1 100644 --- a/vmbox/lib/launch.sh +++ b/vmbox/lib/launch.sh @@ -16,15 +16,30 @@ readonly VMBOX_MEM="${VMBOX_MEM:-4096}" readonly VMBOX_SMP="${VMBOX_SMP:-4}" # $1 = vm name. Echoes the serial-log index it started. -# First VM on the segment listens; later ones connect to it. Plain TCP on -# loopback, so this works where multicast does not. +# VM-to-VM segment, addressed by the VM's own index. +# +# History, so nobody re-tries these: `mcast=` is silently dead on this host +# (`lo` has no MULTICAST flag, no multicast route -- qemu accepts it and drops +# every frame), and a listen=/connect= socket hub connected but never passed +# traffic. `dgram` with explicit UDP endpoints is the third approach: each VM +# binds its own port and points at its peer, so a PAIR of sandboxes can talk. +# This is a pair-wise link, not a switch: VMs 1 and 2 are wired together, 3 +# and 4 are wired together, and so on. _peer_netdev() { - if (exec 3<>/dev/tcp/127.0.0.1/"$VMBOX_HUB_PORT") 2>/dev/null; then - exec 3<&- 2>/dev/null || true - printf 'connect=127.0.0.1:%s' "$VMBOX_HUB_PORT" + # Pair (1,2), (3,4), (5,6)... An index pairs UP when odd and DOWN when + # even. Getting this backwards points a VM at a port nobody bound, which + # looks exactly like a broken network: both ends up, zero frames, ARP + # fails, and nothing in any log says why. + local index="$1" base="$VMBOX_HUB_PORT" mine theirs + mine=$(( base + index )) + if (( index % 2 == 1 )); then + theirs=$(( base + index + 1 )) else - printf 'listen=127.0.0.1:%s' "$VMBOX_HUB_PORT" + theirs=$(( base + index - 1 )) fi + # Type first, then id: qemu rejects `id=` before the netdev type. + printf 'dgram,id=net1,local.type=inet,local.host=127.0.0.1,local.port=%s,remote.type=inet,remote.host=127.0.0.1,remote.port=%s' \ + "$mine" "$theirs" } launch_vm() { @@ -67,7 +82,7 @@ launch_vm() { -device virtio-net-pci,netdev=net0 # Shared segment so sandboxes can reach each other. No root, no bridge. # listen= for the first VM up, connect= for the rest (see _peer_netdev). - -netdev "socket,id=net1,$(_peer_netdev)" + -netdev "$(_peer_netdev "$index")" -device "virtio-net-pci,netdev=net1,mac=52:54:00:be:ef:$(printf '%02x' "$index")" # Repos under test, mounted PHYSICALLY read-only. Scoped to an # explicit share dir -- NEVER $HOME, which would hand the guest