diff --git a/vmbox/--help/sockets.target.wants/sshd-unix-local.socket b/vmbox/--help/sockets.target.wants/sshd-unix-local.socket new file mode 120000 index 0000000..ade6b13 --- /dev/null +++ b/vmbox/--help/sockets.target.wants/sshd-unix-local.socket @@ -0,0 +1 @@ +../sshd-unix-local.socket \ No newline at end of file diff --git a/vmbox/--help/sshd-unix-local.socket b/vmbox/--help/sshd-unix-local.socket new file mode 100644 index 0000000..da3cede --- /dev/null +++ b/vmbox/--help/sshd-unix-local.socket @@ -0,0 +1,11 @@ +# Automatically generated by systemd-ssh-generator + +[Unit] +Description=OpenSSH Server Socket (systemd-ssh-generator, AF_UNIX Local) +Documentation=man:systemd-ssh-generator(8) + +[Socket] +ListenStream=/run/ssh-unix-local/socket +Accept=yes +PollLimitIntervalSec=30s +PollLimitBurst=50 diff --git a/vmbox/--help/sshd-unix-local@.service b/vmbox/--help/sshd-unix-local@.service new file mode 120000 index 0000000..d5942f3 --- /dev/null +++ b/vmbox/--help/sshd-unix-local@.service @@ -0,0 +1 @@ +/usr/lib/systemd/system/sshd@.service \ No newline at end of file diff --git a/vmbox/guest/provision.sh b/vmbox/guest/provision.sh index b6a6a1e..10e70fb 100644 --- a/vmbox/guest/provision.sh +++ b/vmbox/guest/provision.sh @@ -30,6 +30,8 @@ if [[ -f /etc/default/grub ]]; then grub-mkconfig -o /boot/grub/grub.cfg 2>/dev/null || true fi systemctl enable serial-getty@ttyS0.service +# ttyS1 carries the interactive console vmbox drives programmatically. +systemctl enable serial-getty@ttyS1.service # --- ssh ------------------------------------------------------------------ # `systemctl enable sshd` alone is NOT enough on this image: systemd's @@ -37,8 +39,13 @@ systemctl enable serial-getty@ttyS0.service # no-op and leave nothing listening on TCP:22. The guest then boots perfectly # and every ssh gets "Connection reset by peer". Force the real service on and # verify the symlink exists, failing the build loudly if it does not. +# Do NOT `systemctl disable sshd.socket` here. On Arch sshd.socket and +# sshd.service are alternatives, and disabling the socket after enabling the +# service also drops the service's multi-user.target.wants symlink -- the +# build then reports "enabled" while the sealed image has no sshd at boot. +# Evidence: the build log shows "Created symlink" for every other unit but +# never for sshd.service, and the guest boot shows only the AF_UNIX socket. systemctl enable sshd.service -systemctl disable sshd.socket 2>/dev/null || true if [[ ! -e /etc/systemd/system/multi-user.target.wants/sshd.service ]]; then echo "provision: FAILED to enable sshd.service -- guest would be unreachable" >&2 exit 1 @@ -50,6 +57,23 @@ ls /etc/ssh/ssh_host_*_key >/dev/null 2>&1 || { echo "provision: FAILED to generate sshd host keys" >&2 exit 1 } +# Belt and braces. On this image sshd.service is enabled and its dependencies +# are satisfied, yet it produces no start line at boot -- neither started nor +# failed. Rather than keep guessing at the mechanism, force it explicitly once +# multi-user is reached. Harmless if sshd already started. +cat > /etc/systemd/system/vmbox-sshd-kick.service <<'KICK' +[Unit] +Description=vmbox: ensure sshd is actually running +After=multi-user.target network.target +[Service] +Type=oneshot +ExecStart=/usr/bin/systemctl start --no-block sshd.service +RemainAfterExit=yes +[Install] +WantedBy=multi-user.target +KICK +systemctl enable vmbox-sshd-kick.service + echo "provision: sshd.service enabled with $(ls /etc/ssh/ssh_host_*_key | wc -l) host keys" # --- systemd --user session ---------------------------------------------- @@ -62,12 +86,14 @@ loginctl enable-linger "$GUEST_USER" 2>/dev/null || true # Without autologin here an unreachable guest can only be diagnosed by # inferring from absent log lines, which is exactly how a single sshd fault # turned into several blind image rebuilds. -install -d -m 755 /etc/systemd/system/serial-getty@ttyS0.service.d -cat > /etc/systemd/system/serial-getty@ttyS0.service.d/autologin.conf < "/etc/systemd/system/serial-getty@${tty}.service.d/autologin.conf" </dev/null' /dev/null' /dev/null; then + printf 'ssh' + else + printf 'serial' + fi +} + +_run_exec_serial() { + local name="$1" cmd="$2" console + console="$(vm_console "$name")" + [[ -S "$console" ]] || die "no serial console socket for '$name'" + python3 "$VMBOX_LIB_DIR/serial_exec.py" "$console" \ + "{ $cmd ; } ; echo \$? > $RC_PATH" } # Give a guest that is shutting down time to finish writing its serial log. @@ -109,11 +137,22 @@ _run_finish() { } # Boot the sandbox again just far enough to read the recorded exit code. +# The overlay survives the poweroff, so the value written before the machine +# stopped is still there. _run_recover_rc() { - local name="$1" rc="" + local name="$1" rc="" console launch_vm "$name" >/dev/null 2>&1 || return 0 - if vm_wait_ssh "$name" 120; then + + if vm_wait_ssh "$name" 90; then rc="$(vm_ssh_exec "$name" "cat $RC_PATH 2>/dev/null" 2>/dev/null | tr -dc '0-9')" + else + # ssh may be broken or simply slower than the console; the serial + # shell can read the file just as well. + console="$(vm_console "$name")" + if [[ -S "$console" ]]; then + rc="$(python3 "$VMBOX_LIB_DIR/serial_exec.py" "$console" \ + "cat $RC_PATH 2>/dev/null" 2>/dev/null | grep -oE '^[0-9]+' | head -1)" + fi fi printf '%s' "$rc" } diff --git a/vmbox/lib/serial_exec.py b/vmbox/lib/serial_exec.py new file mode 100755 index 0000000..c61636f --- /dev/null +++ b/vmbox/lib/serial_exec.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Run a command in a vmbox guest over its serial console root shell. + +This exists because ssh is not the right channel for the thing vmbox is for. +A shutdown script kills the connection carrying its own result, and the serial +console keeps printing right through the poweroff -- so for the shutdown case +serial is not a fallback, it is the better transport. It also works when sshd +is broken, which makes an unreachable guest debuggable instead of opaque. + +Every command is bracketed by explicit markers and waited on until its END +marker appears. Never use a fixed sleep: if the window closes early the next +command is typed while the previous one still runs, the tty echoes that input, +and the echoed text is indistinguishable from output in the transcript. + +Usage: + serial_exec.py -> runs, prints output + serial_exec.py --wait-prompt -> just wait for a shell +""" + +from __future__ import annotations + +import re +import socket +import sys +import time +import uuid + +PROMPT = "[root@vmbox" +# Strip ANSI CSI sequences and the OSC 3008 shell-integration blocks that +# systemd's login emits, both of which otherwise corrupt marker matching. +_ANSI = re.compile(r"\x1b\[[0-9;?]*[a-zA-Z]") +_OSC = re.compile(r"\x1b?\]3008;[^\x07\\]*(?:\x07|\\)?") + + +def clean(raw: bytes) -> str: + text = raw.decode("utf-8", "replace") + text = _ANSI.sub("", text) + text = _OSC.sub("", text) + return text.replace("\r", "") + + +class Console: + """A line-oriented client for the guest's serial console socket.""" + + def __init__(self, path: str) -> None: + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(path) + self.sock.settimeout(0.4) + self.buf = b"" + + def read_until(self, needle: str, timeout: float) -> bool: + end = time.time() + timeout + while time.time() < end: + try: + chunk = self.sock.recv(4096) + except TimeoutError: + continue + except OSError: + return False + if not chunk: + return False + self.buf += chunk + if needle in clean(self.buf): + return True + return False + + def wait_prompt(self, timeout: float = 180) -> bool: + # Nudge with a newline: if the shell is already up we get a fresh + # prompt immediately instead of waiting for unrelated output. + self.sock.sendall(b"\n") + return self.read_until(PROMPT, timeout) + + def run(self, cmd: str, timeout: float = 300) -> str: + token = uuid.uuid4().hex[:8] + start, done = f"<<>>", f"<<>>" + self.buf = b"" + # `; echo RC=$?` keeps the guest's exit status, which the ssh path + # cannot report once the machine stops. + self.sock.sendall(f"echo {start}; {cmd}; echo RC=$?; echo {done}\n".encode()) + if not self.read_until(done, timeout): + return f"### TIMEOUT: no end marker after {timeout}s ###" + text = clean(self.buf) + body = text.split(start, 1)[-1].split(done, 1)[0] + # Drop the echoed command line itself. + lines = [ln for ln in body.splitlines() if start not in ln and done not in ln] + return "\n".join(lines).strip() + + +def main() -> int: + if len(sys.argv) < 3: + print(__doc__, file=sys.stderr) + return 2 + console = Console(sys.argv[1]) + if sys.argv[2] == "--wait-prompt": + return 0 if console.wait_prompt() else 1 + if not console.wait_prompt(): + print("serial_exec: no root prompt on the console", file=sys.stderr) + return 1 + print(console.run(" ".join(sys.argv[2:]))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vmbox/lib/smoke.sh b/vmbox/lib/smoke.sh index 20c74b9..9942a1c 100644 --- a/vmbox/lib/smoke.sh +++ b/vmbox/lib/smoke.sh @@ -21,7 +21,7 @@ base_smoke_test() { source "$VMBOX_LIB_DIR/launch.sh" source "$VMBOX_LIB_DIR/ssh.sh" - local smoke="__smoke" + local smoke="vmbox-smoke" vm_exists "$smoke" && overlay_rm "$smoke" >/dev/null 2>&1 overlay_new "$smoke" >/dev/null @@ -42,7 +42,7 @@ base_smoke_test() { if (( failed == 0 )); then overlay_rm "$smoke" >/dev/null 2>&1 || true else - warn "keeping sandbox '__smoke' so its logs survive; remove it with: vm rm __smoke" + warn "keeping sandbox 'vmbox-smoke' so its logs survive; remove it with: vm rm vmbox-smoke" die "refusing to ship an unreachable base image" fi }