mirror of
https://github.com/kuhyx/utils.git
synced 2026-11-19 12:34:33 +01:00
The peer-address service read a `vmbox.peer=` kernel argument that the launcher never passed -- the reader was written and the writer forgotten -- so eth1 came up DOWN with no address on every guest. It now derives the address from the NIC's own MAC (the host allocates 52:54:00:be:ef:<index>), which removes the coupling entirely: nothing has to be passed in, so the launcher and the guest cannot disagree about which index a VM has. That fixed the addressing but NOT the connectivity. `mcast=` cannot work on this host at all -- `lo` carries no MULTICAST flag and there is no multicast route, so qemu accepts the option and silently drops every frame, which is the same "flags accepted, nothing happens" trap as POWERDOWN. A listen=/connect= socket hub over loopback TCP was tried next: the hub port listens, both guests bring eth1 up with the right addresses, and ARP still fails. Documented as a known limitation rather than papered over. Sandboxes work fully on their own; two guests cannot talk to each other yet. Also corrects the destructive demo: a running Linux SURVIVES `rm -rf /usr` because processes hold their binaries through open file handles, so sshd keeps answering. The honest proof of destruction is that the machine cannot boot again, so the demo now reboots it and asserts it does not come back. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VdyMBGCrGVGDWtJ9xe48f3
209 lines
9.3 KiB
Bash
209 lines
9.3 KiB
Bash
#!/bin/bash
|
|
# ============================================================================
|
|
# Runs INSIDE the guest during `vm build`, as root, exactly once.
|
|
# Installs the test surface (Xorg/i3, dev tools), wires serial+autologin,
|
|
# then leaves the image ready to be sealed.
|
|
# ============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
GUEST_USER="${1:-arch}"
|
|
|
|
echo "=== vmbox provision: starting ==="
|
|
|
|
# The cloud image ships an empty package DB; -Syu also picks up any security
|
|
# fixes since the image was cut.
|
|
pacman -Syu --noconfirm --needed \
|
|
base-devel git openssh sudo python python-pip \
|
|
xorg-server xorg-xinit xorg-xrandr xorg-xset i3-wm xterm dmenu \
|
|
strace jq bats shellcheck kcov rsync inetutils \
|
|
iproute2 nftables which less vim
|
|
|
|
# --- serial console -------------------------------------------------------
|
|
# The verdict logic reads the serial log to tell a clean poweroff from one
|
|
# that died mid-sequence, so the guest must actually WRITE its console there.
|
|
# We keep the VGA device for screenshots, so we cannot rely on -nographic's
|
|
# implicit redirect: request ttyS0 explicitly on the kernel cmdline.
|
|
if [[ -f /etc/default/grub ]]; then
|
|
sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT="\(.*\)"/GRUB_CMDLINE_LINUX_DEFAULT="\1 console=tty0 console=ttyS0,115200"/' \
|
|
/etc/default/grub
|
|
grub-mkconfig -o /boot/grub/grub.cfg 2>/dev/null || true
|
|
fi
|
|
systemctl enable serial-getty@ttyS0.service
|
|
# ttyS1 carries the interactive console vmbox drives programmatically.
|
|
systemctl enable serial-getty@ttyS1.service
|
|
|
|
# --- ssh ------------------------------------------------------------------
|
|
# `systemctl enable sshd` alone is NOT enough on this image: systemd's
|
|
# ssh-generator provides a socket-activated AF_UNIX unit, so the enable can
|
|
# no-op and leave nothing listening on TCP:22. The guest then boots perfectly
|
|
# and every ssh gets "Connection reset by peer". Force the real service on and
|
|
# verify the symlink exists, failing the build loudly if it does not.
|
|
# Do NOT `systemctl disable sshd.socket` here. On Arch sshd.socket and
|
|
# sshd.service are alternatives, and disabling the socket after enabling the
|
|
# service also drops the service's multi-user.target.wants symlink -- the
|
|
# build then reports "enabled" while the sealed image has no sshd at boot.
|
|
# Evidence: the build log shows "Created symlink" for every other unit but
|
|
# never for sshd.service, and the guest boot shows only the AF_UNIX socket.
|
|
systemctl enable sshd.service
|
|
if [[ ! -e /etc/systemd/system/multi-user.target.wants/sshd.service ]]; then
|
|
echo "provision: FAILED to enable sshd.service -- guest would be unreachable" >&2
|
|
exit 1
|
|
fi
|
|
# Host keys: sshd is "enabled" but FAILS TO START without them, which looks
|
|
# identical to a network fault from the host side. Generate now and assert.
|
|
ssh-keygen -A
|
|
ls /etc/ssh/ssh_host_*_key >/dev/null 2>&1 || {
|
|
echo "provision: FAILED to generate sshd host keys" >&2
|
|
exit 1
|
|
}
|
|
# Belt and braces. On this image sshd.service is enabled and its dependencies
|
|
# are satisfied, yet it produces no start line at boot -- neither started nor
|
|
# failed. Rather than keep guessing at the mechanism, force it explicitly once
|
|
# multi-user is reached. Harmless if sshd already started.
|
|
cat > /etc/systemd/system/vmbox-sshd-kick.service <<'KICK'
|
|
[Unit]
|
|
Description=vmbox: ensure sshd is actually running
|
|
After=multi-user.target network.target
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/bin/systemctl start --no-block sshd.service
|
|
RemainAfterExit=yes
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
KICK
|
|
systemctl enable vmbox-sshd-kick.service
|
|
|
|
echo "provision: sshd.service enabled with $(ls /etc/ssh/ssh_host_*_key | wc -l) host keys"
|
|
|
|
# --- systemd --user session ----------------------------------------------
|
|
# Several target scripts use `systemctl --user`, which needs a real session
|
|
# bus. Lingering gives the user one without an interactive login.
|
|
loginctl enable-linger "$GUEST_USER" 2>/dev/null || true
|
|
|
|
# --- serial console autologin -------------------------------------------
|
|
# Debuggability: when ssh is broken, the serial console is the ONLY way in.
|
|
# Without autologin here an unreachable guest can only be diagnosed by
|
|
# inferring from absent log lines, which is exactly how a single sshd fault
|
|
# turned into several blind image rebuilds.
|
|
for tty in ttyS0 ttyS1; do
|
|
install -d -m 755 "/etc/systemd/system/serial-getty@${tty}.service.d"
|
|
cat > "/etc/systemd/system/serial-getty@${tty}.service.d/autologin.conf" <<SUNIT
|
|
[Service]
|
|
ExecStart=
|
|
ExecStart=-/sbin/agetty --autologin root --keep-baud 115200,57600,38400,9600 %I \$TERM
|
|
SUNIT
|
|
done
|
|
|
|
# --- host repo share ------------------------------------------------------
|
|
# Mounted read-only; the guest clones out of it rather than working in it.
|
|
install -d -m 755 /mnt/hostrepo
|
|
if ! grep -q hostrepo /etc/fstab 2>/dev/null; then
|
|
echo 'hostrepo /mnt/hostrepo 9p trans=virtio,version=9p2000.L,ro,nofail 0 0' >> /etc/fstab
|
|
fi
|
|
|
|
# --- X11 autologin + i3 ---------------------------------------------------
|
|
# Locker tests need a real X session on tty1. Autologin then startx.
|
|
install -d -m 755 /etc/systemd/system/getty@tty1.service.d
|
|
cat > /etc/systemd/system/getty@tty1.service.d/autologin.conf <<UNIT
|
|
[Service]
|
|
ExecStart=
|
|
ExecStart=-/sbin/agetty --autologin ${GUEST_USER} --noclear %I \$TERM
|
|
UNIT
|
|
|
|
cat > "/home/$GUEST_USER/.xinitrc" <<'XINIT'
|
|
exec i3
|
|
XINIT
|
|
chown "$GUEST_USER:$GUEST_USER" "/home/$GUEST_USER/.xinitrc"
|
|
|
|
# startx on tty1 login only, so `vm ssh` sessions do not try to start X.
|
|
cat > "/home/$GUEST_USER/.bash_profile" <<'PROFILE'
|
|
[[ -f ~/.bashrc ]] && . ~/.bashrc
|
|
if [[ -z ${DISPLAY:-} && $(tty) == /dev/tty1 ]]; then
|
|
exec startx -- -keeptty >~/.xsession.log 2>&1
|
|
fi
|
|
PROFILE
|
|
chown "$GUEST_USER:$GUEST_USER" "/home/$GUEST_USER/.bash_profile"
|
|
|
|
# i3 without a config prompts a wizard that blocks the session; ship a minimal one.
|
|
install -d -m 755 "/home/$GUEST_USER/.config/i3"
|
|
cat > "/home/$GUEST_USER/.config/i3/config" <<'I3'
|
|
set $mod Mod4
|
|
font pango:monospace 10
|
|
bindsym $mod+Return exec xterm
|
|
bindsym $mod+d exec dmenu_run
|
|
bindsym $mod+Shift+q kill
|
|
# Marker window so screenshot tests have something deterministic to match.
|
|
exec --no-startup-id xterm -T vmbox-ready -e 'echo VMBOX READY; exec bash'
|
|
I3
|
|
chown -R "$GUEST_USER:$GUEST_USER" "/home/$GUEST_USER/.config"
|
|
|
|
# --- static IP on the VM-to-VM segment ------------------------------------
|
|
# The multicast segment has no DHCP (the user-mode NIC serves the other
|
|
# interface), so without a static address sandboxes cannot reach each other.
|
|
# The last octet is injected per-VM at launch via the vmbox.peer kernel arg.
|
|
cat > /usr/local/bin/vmbox-peer-ip <<'PEER'
|
|
#!/bin/bash
|
|
# Bring up the VM-to-VM segment and give this guest a static address.
|
|
#
|
|
# The address comes from the NIC's own MAC (the host allocates
|
|
# 52:54:00:be:ef:<index>), NOT from a kernel argument: deriving it here means
|
|
# the guest needs nothing passed in, so there is no way for the launcher and
|
|
# the guest to disagree about which index this VM has.
|
|
set -euo pipefail
|
|
subnet="${VMBOX_SUBNET:-10.77.0}"
|
|
for path in /sys/class/net/*/address; do
|
|
dev="$(basename "$(dirname "$path")")"
|
|
[[ "$dev" == lo ]] && continue
|
|
mac="$(cat "$path")"
|
|
# Only the peer NIC carries the be:ef prefix; the primary nic must not be
|
|
# touched, since it carries ssh.
|
|
[[ "$mac" == 52:54:00:be:ef:* ]] || continue
|
|
idx=$((16#${mac##*:}))
|
|
ip addr add "${subnet}.${idx}/24" dev "$dev" 2>/dev/null || true
|
|
ip link set "$dev" up
|
|
exit 0
|
|
done
|
|
PEER
|
|
chmod 755 /usr/local/bin/vmbox-peer-ip
|
|
chmod 755 /usr/local/bin/vmbox-peer-ip
|
|
|
|
cat > /etc/systemd/system/vmbox-peer-ip.service <<'PSVC'
|
|
[Unit]
|
|
Description=vmbox peer network address
|
|
After=network.target
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/bin/vmbox-peer-ip
|
|
RemainAfterExit=yes
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
PSVC
|
|
systemctl enable vmbox-peer-ip.service
|
|
|
|
systemctl enable systemd-networkd systemd-resolved 2>/dev/null || true
|
|
|
|
# --- keep the pinned clock pinned -----------------------------------------
|
|
# The host passes -rtc base=<ts> so time-gated scripts (the shutdown installers
|
|
# key off a 21:00-05:00 window) are deterministic. systemd-timesyncd reaches
|
|
# the network on boot and silently overwrites that with real time, which
|
|
# re-introduces exactly the nondeterminism -rtc exists to remove.
|
|
systemctl disable systemd-timesyncd 2>/dev/null || true
|
|
systemctl mask systemd-timesyncd 2>/dev/null || true
|
|
# CRITICAL: systemd-time-wait-sync waits for a sync that can never arrive once
|
|
# timesyncd is masked. It then sits in "start running" forever and every other
|
|
# job -- sshd included -- queues behind it as "start waiting", so the system
|
|
# never reaches multi-user.target. The guest boots to a login prompt and looks
|
|
# healthy while being permanently unreachable. Mask it together with timesyncd.
|
|
systemctl disable systemd-time-wait-sync 2>/dev/null || true
|
|
systemctl mask systemd-time-wait-sync 2>/dev/null || true
|
|
# Do NOT write a /etc/systemd/network/*.network file here: an en* match
|
|
# also catches the primary NIC and outranks the cloud image's own DHCP
|
|
# config, which breaks sshd's network and makes the sandbox unreachable.
|
|
# (Learned the hard way -- it cost a full rebuild.) Masking the unit above is
|
|
# sufficient to keep the pinned clock; UseNTP only matters if timesyncd runs.
|
|
# cloud-init also writes an NTP config on some images; neutralise it.
|
|
rm -f /etc/systemd/timesyncd.conf.d/*.conf 2>/dev/null || true
|
|
|
|
echo "=== vmbox provision: done ==="
|