diff --git a/docs/superpowers/evidence/boot-repair-esp-recovery-20260730.json b/docs/superpowers/evidence/boot-repair-esp-recovery-20260730.json new file mode 100644 index 00000000..6ed947d4 --- /dev/null +++ b/docs/superpowers/evidence/boot-repair-esp-recovery-20260730.json @@ -0,0 +1,67 @@ +{ + "intent": "Provide an offline, self-contained recovery tool that repairs a stale EFI System Partition from the systemd emergency shell, and prevent the failure recurring by gating kernel transactions on a mounted ESP.", + "scope": [ + "linux_configuration/scripts/boot_recovery/boot-repair", + "linux_configuration/scripts/boot_recovery/install.sh", + "linux_configuration/scripts/boot_recovery/README.md", + "linux_configuration/scripts/boot_recovery/hooks/05-boot-mounted-guard.hook", + "linux_configuration/scripts/boot_recovery/hooks/99-boot-autorepair.hook", + "linux_configuration/scripts/boot_recovery/tests/test_boot_repair.sh", + "linux_configuration/scripts/boot_recovery/tests/live_test.sh", + "Non-goal: no network access at any point; recovery uses only /var/cache/pacman/pkg", + "Non-goal: does not source linux_configuration/scripts/lib/common.sh - the tool must stand alone in an emergency shell" + ], + "changes": [ + "boot-repair: detects and repairs the 2026-07-30 emergency-mode failure chain - stale ESP kernel, gutted module tree, unloadable vfat, unmounted ESP, missing modules.dep, shadow kernel files written into the root filesystem's /boot, noauto/nofail fstab drift, and ParallelDownloads > 1 (the alpm-hook segfault trigger that was the root cause).", + "Treats a module tree as usable only when BOTH vmlinuz and kernel/ exist, so the metadata-only wreckage the bug leaves behind is never selected as the target kernel.", + "Safety guards on the only destructive path: files under /boot are deleted only while it is not a mountpoint AND contains no EFI/ or loader/. EFI/ and loader/ are never written or removed.", + "Recovers vfat offline by extracting fs/fat + fs/nls from the cached kernel package, since mounting the ESP requires the very module the failed upgrade deleted.", + "05-boot-mounted-guard.hook: PreTransaction gate with AbortOnFail that refuses a kernel install when the ESP is unmounted.", + "99-boot-autorepair.hook: PostTransaction auto-repair that fixes ESP drift before the machine is ever rebooted, exiting non-zero if it cannot.", + "--root flag makes the logic testable against fixture trees and usable from an Arch ISO chroot.", + "install.sh verifies its dependencies, installs to /usr/local/sbin (root filesystem - the ESP is unreachable when this is needed), and runs the installed copy to prove it works." + ], + "verification": [ + { + "command": "./tests/test_boot_repair.sh", + "result": "pass", + "evidence": "31 passed, 0 failed. Covers healthy system, stale ESP kernel detect+repair, gutted-tree rejection, newest-complete selection, fstab noauto/nofail reset with backup, shadow-file deletion, the refuse-to-delete-a-real-ESP guard, ParallelDownloads fix, missing modules.dep, no-kernel guidance, CLI contract." + }, + { + "command": "sudo ./tests/live_test.sh", + "result": "pass", + "evidence": "11 passed, 0 failed on the real machine. Unmounted the real ESP, confirmed the preflight gate opens when mounted and closes when not, confirmed dry-run writes nothing, planted shadow files and confirmed they were deleted and the ESP then mounted with EFI/, loader/ and kernel 7.1.5-arch1-2 intact. EXIT trap remounts the ESP regardless of outcome." + }, + { + "command": "sudo pacman -U /var/cache/pacman/pkg/linux-7.1.5.arch1-2-x86_64.pkg.tar.zst (ESP mounted)", + "result": "pass", + "evidence": "(1/3) Verifying the ESP is mounted before installing a kernel... boot-repair: ESP mounted at /boot - proceeding. (6/6) Verifying the ESP received the new kernel... Both hooks fired; post-transaction hook silent, meaning nothing needed repair." + }, + { + "command": "sudo umount /boot && sudo pacman -U .../linux-7.1.5.arch1-2-x86_64.pkg.tar.zst", + "result": "pass", + "evidence": "==> boot-repair: ABORTING TRANSACTION / error: failed to commit transaction (failed to run transaction hooks) / Errors occurred, no packages were upgraded. Nothing was written into the unmounted /boot. The gate held." + }, + { + "command": "shellcheck boot-repair install.sh tests/*.sh", + "result": "pass", + "evidence": "Clean. SC2034/SC2155/SC2115/SC2001 fixed at source rather than suppressed." + }, + { + "command": "shfmt -d -i 2 -ci -sr -s boot-repair install.sh tests/*.sh", + "result": "pass", + "evidence": "Clean, matching the options used by scripts/meta/shell_check.sh." + } + ], + "risks": [ + "The PreTransaction hook uses AbortOnFail, so a bug in boot-repair --preflight would block all kernel upgrades. Mitigated by the live test proving both the open and closed paths, and by --preflight being a single findmnt check.", + "The 99-boot-autorepair hook is itself forked by alpm, so it is subject to the same ParallelDownloads segfault it compensates for. This is why boot-repair also enforces ParallelDownloads = 1 rather than relying on the hook alone.", + "Shadow-file deletion is destructive. Guarded by requiring /boot to be unmounted AND free of EFI/ and loader/; both guards are covered by fixture and live tests.", + "If the running kernel's package is absent from /var/cache/pacman/pkg, vfat cannot be restored offline. The script reports this and prints Arch ISO chroot recovery steps instead of guessing." + ], + "rollback": [ + "rm /etc/pacman.d/hooks/05-boot-mounted-guard.hook /etc/pacman.d/hooks/99-boot-autorepair.hook to remove the gates.", + "rm /usr/local/sbin/boot-repair to remove the tool.", + "After rollback validate that a kernel transaction still completes: sudo pacman -U /var/cache/pacman/pkg/linux-*.pkg.tar.zst, then confirm findmnt /boot shows the ESP and the version in /boot/vmlinuz-linux has a matching /usr/lib/modules// directory." + ] +} diff --git a/linux_configuration/scripts/boot_recovery/README.md b/linux_configuration/scripts/boot_recovery/README.md new file mode 100644 index 00000000..0f1ae754 --- /dev/null +++ b/linux_configuration/scripts/boot_recovery/README.md @@ -0,0 +1,115 @@ +# boot-repair + +Offline recovery for a stale EFI System Partition — the failure that dropped +this laptop into emergency mode on 2026-07-30. + +## The failure it fixes + +A kernel upgrade replaced `/usr/lib/modules/` with ``, but the new +kernel image never reached the ESP. The firmware kept booting the **old** +`vmlinuz`, whose module tree no longer existed, so nothing modular could load +after pivot-root. `vfat` was among the casualties — which meant `/boot` itself +could not be mounted, `local-fs.target` failed, and systemd dropped to +emergency mode. pacman reported the upgrade as successful. + +**Root cause on this machine:** `pacman-git` with `ParallelDownloads > 1` makes +every forked alpm hook child segfault (SIGSEGV in libc), silently skipping +`90-mkinitcpio-install` and `depmod`. + +## Install + +```bash +sudo ./install.sh +``` + +Installs: + +| Path | Purpose | +|---|---| +| `/usr/local/sbin/boot-repair` | the repair tool | +| `/etc/pacman.d/hooks/05-boot-mounted-guard.hook` | pre-transaction **gate** | +| `/etc/pacman.d/hooks/99-boot-autorepair.hook` | post-transaction auto-repair | + +It lives on the **root** filesystem by necessity: the ESP is unreachable by +definition when this is needed. + +## Use + +```bash +sudo boot-repair # repair now (default) +sudo boot-repair --dry-run # report only, change nothing +``` + +From the systemd emergency shell the root filesystem is already mounted, so: + +``` +/usr/local/sbin/boot-repair +``` + +No network is used at any point. Missing kernel modules are recovered from +`/var/cache/pacman/pkg`. + +### Options + +| Flag | Meaning | +|---|---| +| `--dry-run` | Report problems, change nothing | +| `--preflight` | Exit non-zero unless the ESP is mounted (used by the gate hook) | +| `--auto` | Quiet repair for the post-transaction hook | +| `--root DIR` | Operate on another root (fixtures, or an Arch ISO chroot) | +| `--esp DEVICE` | Override ESP auto-detection | + +Exit status: `0` consistent/repaired, `1` problems remain, `2` usage error. + +## What it checks and repairs + +1. Root filesystem read-only after a failed boot → remount rw +2. `noauto`/`nofail` on `/boot` in fstab → reset to `defaults` (backed up). + `nofail` turns a loud emergency drop into a *silent* boot of a stale kernel, + which is strictly worse. +3. Orphaned kernel files written into the unmounted `/boot` → delete +4. `vfat` unloadable → restore `fs/fat` + `fs/nls` from the cached kernel + package, `depmod`, `modprobe` +5. ESP not mounted → mount it +6. Missing `modules.dep` → `depmod` +7. ESP kernel ≠ newest complete module tree → install the kernel +8. initramfs ≠ that kernel → `mkinitcpio -P` +9. `ParallelDownloads > 1` → set to `1` (the root cause) + +A module tree counts as usable only if it has **both** `vmlinuz` and `kernel/`. +The wreckage this bug leaves behind is a directory holding only `modules.*` +metadata, and a naive "newest directory" pick would choose it. + +## Safety invariants + +- Files under `/boot` are deleted **only** while it is not a mountpoint **and** + contains no `EFI/` or `loader/`. Deleting after mounting would destroy the + real kernel. +- `EFI/` and `loader/` are never written or removed. +- The running kernel's module tree is never deleted. +- No network, ever. + +## Tests + +```bash +./tests/test_boot_repair.sh # 31 fixture tests, no root needed +sudo ./tests/live_test.sh # 11 live tests, reversible +``` + +`live_test.sh` refuses to start unless the system is already consistent, and an +EXIT trap remounts the ESP however it ends. It covers what fixtures cannot: +the pre-transaction gate, unmounting the real ESP, and deleting shadow files +off the real root filesystem while proving the ESP survives. + +### Verified behaviour + +- 31/31 fixture tests, 11/11 live tests, `shellcheck` clean +- Gate proven to abort a real kernel transaction with the ESP unmounted: + `error: failed to commit transaction`, *no packages upgraded*, nothing + written into the unmounted `/boot` + +## Limitation + +If the running kernel's package is not in `/var/cache/pacman/pkg`, step 4 is +impossible offline. The script says so and prints the Arch-ISO chroot recovery +commands rather than guessing. diff --git a/linux_configuration/scripts/boot_recovery/boot-repair b/linux_configuration/scripts/boot_recovery/boot-repair new file mode 100755 index 00000000..0de17c2c --- /dev/null +++ b/linux_configuration/scripts/boot_recovery/boot-repair @@ -0,0 +1,732 @@ +#!/bin/bash + +# ============================================================================ +# boot-repair — offline repair for a stale/unwritten EFI System Partition +# ============================================================================ +# +# The failure this fixes (seen 2026-07-30): +# A kernel upgrade replaced /usr/lib/modules/ with , but the new +# kernel image never reached the ESP. The firmware kept booting the OLD +# vmlinuz, whose module tree no longer existed — so nothing modular could +# load after pivot-root. In particular `vfat` was gone, /boot itself could +# not be mounted, local-fs.target failed and systemd dropped to emergency +# mode. The upgrade was reported as successful. +# +# Root cause on this machine: pacman-git with ParallelDownloads > 1 makes +# every forked alpm hook child segfault, silently skipping the +# 90-mkinitcpio-install and depmod hooks. +# +# Design constraints (deliberate): +# * SELF-CONTAINED — no sourcing of repo libs. It must run from the systemd +# emergency shell where only the root filesystem is mounted. +# * FULLY OFFLINE — never touches the network. Missing kernel modules are +# recovered from /var/cache/pacman/pkg, nothing else. +# * Repairs by default. Use --dry-run to inspect without changing anything. +# +# Safety invariants: +# * Files under /boot are ONLY deleted while /boot is not a mountpoint AND +# contains no EFI/ or loader/ directory. Deleting them after mounting +# would destroy the real kernel and leave the machine unbootable. +# * EFI/ and loader/ on the ESP are never written or removed. +# * The running kernel's module tree is never deleted. +# ============================================================================ + +set -euo pipefail + +# Emergency shells can have a threadbare PATH. +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH + +readonly SCRIPT_NAME="${0##*/}" +readonly ESP_MOUNTPOINT="/boot" + +# Defaults +ROOT="/" +MODE="fix" # fix | dry-run | preflight | auto +ESP_DEV_OVERRIDE="" +PROBLEMS=0 +REPAIRS=0 +QUIET=0 + +# ---------------------------------------------------------------------------- +# Output helpers +# ---------------------------------------------------------------------------- + +msg() { [[ $QUIET -eq 1 ]] || printf '%s\n' "$*"; } +ok() { [[ $QUIET -eq 1 ]] || printf ' [ OK ] %s\n' "$*"; } +info() { [[ $QUIET -eq 1 ]] || printf ' [INFO] %s\n' "$*"; } + +# A problem that was found. Counted so the exit status can reflect reality. +problem() { + PROBLEMS=$((PROBLEMS + 1)) + printf ' [FAIL] %s\n' "$*" >&2 +} + +warn() { printf ' [WARN] %s\n' "$*" >&2; } + +# A repair that was actually carried out. +repaired() { + REPAIRS=$((REPAIRS + 1)) + printf ' [FIXED] %s\n' "$*" +} + +die() { + printf '%s: error: %s\n' "$SCRIPT_NAME" "$*" >&2 + exit 2 +} + +# True when we are allowed to modify the system. +acting() { [[ $MODE == "fix" || $MODE == "auto" ]]; } + +# Describe an action that dry-run would skip. +would() { msg " [DRY ] would $*"; } + +usage() { + cat << EOF +Usage: $SCRIPT_NAME [OPTIONS] + +Repairs a stale EFI System Partition after a kernel upgrade failed to write +it. Repairs by default; runs fully offline. + +Options: + --dry-run Report problems, change nothing. + --preflight Exit non-zero unless the ESP is mounted. For a pacman + PreTransaction hook that gates kernel upgrades. + --auto Quiet repair for a pacman PostTransaction hook. Prints + only when it acts or fails. + --root DIR Operate on an alternate root (testing, or an Arch ISO + chroot). Live-only steps are skipped. + --esp DEVICE Use DEVICE as the ESP instead of auto-detecting it. + -h, --help Show this help. + +Exit status: + 0 system consistent (or fully repaired) + 1 problems remain / preflight gate failed + 2 usage or environment error +EOF + exit 0 +} + +# ---------------------------------------------------------------------------- +# Path helpers — everything is expressed relative to \$ROOT so the logic can be +# exercised against a fixture tree without touching the real machine. +# ---------------------------------------------------------------------------- + +# Join $ROOT with an absolute path, avoiding a doubled leading slash. +rpath() { + local p="$1" + if [[ $ROOT == "/" ]]; then + printf '%s' "$p" + else + printf '%s%s' "${ROOT%/}" "$p" + fi +} + +# True when operating on the live system (so mount/modprobe are meaningful). +is_live() { [[ $ROOT == "/" ]]; } + +# ---------------------------------------------------------------------------- +# Kernel discovery +# ---------------------------------------------------------------------------- + +# Read the version banner out of a kernel image. Works on a bare file with no +# tooling beyond grep, which matters in a degraded emergency shell. +# Usage: kernel_image_version -> prints e.g. 7.1.5-arch1-2, or nothing +kernel_image_version() { + local img="$1" + [[ -r $img ]] || return 1 + grep -aoE '[0-9]+\.[0-9]+\.[0-9]+-arch[0-9]+-[0-9]+' "$img" 2> /dev/null | head -1 +} + +# A module tree is "complete" only if it has both the kernel image and actual +# modules. The failure mode leaves behind a directory holding just the +# modules.* metadata files, which must not be mistaken for a usable kernel. +kernel_tree_complete() { + local dir="$1" + [[ -f $dir/vmlinuz ]] || return 1 + [[ -d $dir/kernel ]] || return 1 + return 0 +} + +# Newest complete module tree under $ROOT/usr/lib/modules, by version sort. +newest_complete_kernel() { + local moddir kver best="" + moddir="$(rpath /usr/lib/modules)" + [[ -d $moddir ]] || return 1 + while IFS= read -r kver; do + [[ -n $kver ]] || continue + kernel_tree_complete "$moddir/$kver" || continue + best="$kver" + done < <(find "$moddir" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2> /dev/null | sort -V) + [[ -n $best ]] || return 1 + printf '%s' "$best" +} + +# ---------------------------------------------------------------------------- +# ESP discovery +# ---------------------------------------------------------------------------- + +# The fstab spec for /boot (e.g. "UUID=8A14-F8A7"), if present. +fstab_boot_spec() { + local fstab spec mnt + fstab="$(rpath /etc/fstab)" + [[ -r $fstab ]] || return 1 + while read -r spec mnt _; do + [[ $spec == \#* ]] && continue + [[ -z ${spec:-} ]] && continue + if [[ $mnt == "$ESP_MOUNTPOINT" ]]; then + printf '%s' "$spec" + return 0 + fi + done < "$fstab" + return 1 +} + +# The mount options field for /boot in fstab. +fstab_boot_options() { + local fstab spec mnt opts + fstab="$(rpath /etc/fstab)" + [[ -r $fstab ]] || return 1 + while read -r spec mnt _ opts _; do + [[ $spec == \#* ]] && continue + [[ -z ${spec:-} ]] && continue + if [[ $mnt == "$ESP_MOUNTPOINT" ]]; then + printf '%s' "$opts" + return 0 + fi + done < "$fstab" + return 1 +} + +# Resolve the fstab spec to a block device. Prefers /dev/disk/by-uuid so it +# works without blkid, which may be unavailable in a minimal shell. +resolve_esp_device() { + local spec uuid dev + if [[ -n $ESP_DEV_OVERRIDE ]]; then + printf '%s' "$ESP_DEV_OVERRIDE" + return 0 + fi + spec="$(fstab_boot_spec)" || return 1 + case "$spec" in + UUID=*) + uuid="${spec#UUID=}" + if [[ -e /dev/disk/by-uuid/$uuid ]]; then + dev="$(readlink -f "/dev/disk/by-uuid/$uuid")" + printf '%s' "$dev" + return 0 + fi + # Fall back to blkid when the by-uuid symlink is absent. + if command -v blkid > /dev/null 2>&1; then + dev="$(blkid -U "$uuid" 2> /dev/null)" && [[ -n $dev ]] && { + printf '%s' "$dev" + return 0 + } + fi + return 1 + ;; + /dev/*) + printf '%s' "$spec" + return 0 + ;; + *) + return 1 + ;; + esac +} + +# ---------------------------------------------------------------------------- +# Mount state +# ---------------------------------------------------------------------------- + +boot_is_mounted() { + is_live || return 1 + findmnt -n "$ESP_MOUNTPOINT" > /dev/null 2>&1 +} + +# ---------------------------------------------------------------------------- +# Repair steps +# ---------------------------------------------------------------------------- + +# The root filesystem is sometimes left read-only after a failed boot; every +# later step would fail confusingly without this. +ensure_root_writable() { + is_live || return 0 + local testfile + testfile="$(rpath /.boot-repair-write-test)" + if (: > "$testfile") 2> /dev/null; then + rm -f "$testfile" + return 0 + fi + problem "root filesystem is read-only" + if acting; then + mount -o remount,rw / || die "could not remount / read-write" + repaired "remounted / read-write" + else + would "remount / read-write" + fi +} + +# Restore just enough of the RUNNING kernel's module tree to mount a vfat ESP. +# Without this the repair cannot even reach the ESP: mounting it needs vfat, +# and vfat lives in the module tree the upgrade deleted. +ensure_vfat() { + is_live || return 0 + if modprobe -n vfat > /dev/null 2>&1 && modprobe vfat > /dev/null 2>&1; then + ok "vfat available" + return 0 + fi + + problem "vfat cannot be loaded for the running kernel ($(uname -r))" + if ! acting; then + would "restore fat/nls modules from the pacman cache" + return 1 + fi + + local kver pkgver cache pkg + kver="$(uname -r)" + # uname 7.0.5-arch1-1 -> package 7.0.5.arch1-1 + pkgver="${kver/-arch/.arch}" + cache="$(rpath /var/cache/pacman/pkg)" + + pkg="" + if [[ -d $cache ]]; then + # Newest matching cached kernel package for the running version. + pkg="$(find "$cache" -maxdepth 1 -name "linux*-${pkgver}-*.pkg.tar.zst" 2> /dev/null | sort -V | tail -1)" + fi + if [[ -z $pkg ]]; then + problem "no cached package for the running kernel in $cache" + msg "" + msg " Cannot restore vfat offline. Recover with an Arch ISO:" + msg " mount /dev/ /mnt -o subvol=@ && arch-chroot /mnt" + msg " pacman -S linux && $SCRIPT_NAME" + return 1 + fi + + info "restoring fat/nls modules from ${pkg##*/}" + local tmp + tmp="$(mktemp -d)" || die "mktemp failed" + # shellcheck disable=SC2064 # expand $tmp now, on purpose + trap "rm -rf '$tmp'" RETURN + + # Extract only the filesystem modules needed to read a FAT32 ESP. + tar --zstd -xf "$pkg" -C "$tmp" --wildcards \ + "usr/lib/modules/${kver}/kernel/fs/fat/*" \ + "usr/lib/modules/${kver}/kernel/fs/nls/*" 2> /dev/null || + { + problem "could not extract fat/nls modules from ${pkg##*/}" + return 1 + } + + local src dest + src="$tmp/usr/lib/modules/${kver}/kernel" + dest="$(rpath "/usr/lib/modules/${kver}")" + [[ -d $src ]] || { + problem "cached package did not contain fat/nls modules" + return 1 + } + mkdir -p "$dest" + cp -a "$src" "$dest/" + depmod "$kver" 2> /dev/null || true + if modprobe vfat > /dev/null 2>&1; then + repaired "restored vfat for the running kernel" + return 0 + fi + problem "vfat still will not load after restore" + return 1 +} + +# fstab must GATE on the ESP. `nofail` turns this failure into a silent boot of +# a stale kernel, which is strictly worse: the machine looks fine until the +# module tree disappears, one upgrade later. +check_fstab_options() { + local opts fstab + fstab="$(rpath /etc/fstab)" + opts="$(fstab_boot_options)" || { + problem "no /boot entry in $fstab" + return 1 + } + if [[ $opts != *noauto* && $opts != *nofail* ]]; then + ok "fstab gates on /boot ($opts)" + return 0 + fi + problem "fstab has noauto/nofail on /boot ($opts) — failures would be silent" + if ! acting; then + would "rewrite the /boot options to 'defaults'" + return 1 + fi + cp -a "$fstab" "${fstab}.bak-$(date +%Y%m%d-%H%M%S)" + # Rewrite only the options field of the /boot line. + awk -v mp="$ESP_MOUNTPOINT" ' + /^[[:space:]]*#/ { print; next } + NF >= 4 && $2 == mp { $4 = "defaults"; print; next } + { print } + ' OFS='\t' "$fstab" > "${fstab}.new" + mv "${fstab}.new" "$fstab" + repaired "fstab /boot options reset to 'defaults'" + is_live && systemctl daemon-reload 2> /dev/null || true + return 0 +} + +# Orphaned kernel files written into the ROOT filesystem's /boot directory +# while the ESP was unmounted. They shadow the real ESP and waste ~250 MB. +# +# This is the single most dangerous step in the script: doing it after mounting +# would delete the real kernel. Hence the two hard guards below. +clean_shadow_files() { + local bootdir + bootdir="$(rpath $ESP_MOUNTPOINT)" + [[ -d $bootdir ]] || return 0 + + # GUARD 1: never touch a mounted /boot. + if boot_is_mounted; then + return 0 + fi + # GUARD 2: a real ESP always has EFI/ or loader/. If either is present in + # something we believe is unmounted, our mount detection is wrong — stop. + if [[ -d $bootdir/EFI || -d $bootdir/loader ]]; then + warn "unmounted $ESP_MOUNTPOINT contains EFI/ or loader/ — refusing to delete anything" + return 1 + fi + + local -a shadows=() + local f + while IFS= read -r f; do + [[ -n $f ]] && shadows+=("$f") + done < <(find "$bootdir" -maxdepth 1 -type f \ + \( -name 'vmlinuz-*' -o -name 'initramfs-*.img' -o -name '*-ucode.img' \) 2> /dev/null) + + [[ ${#shadows[@]} -gt 0 ]] || return 0 + + problem "${#shadows[@]} orphaned kernel file(s) in the unmounted $ESP_MOUNTPOINT (shadowing the ESP)" + if ! acting; then + would "delete: ${shadows[*]##*/}" + return 1 + fi + rm -f "${shadows[@]}" + repaired "removed ${#shadows[@]} shadow file(s) from the root filesystem" + return 0 +} + +mount_esp() { + is_live || return 0 + if boot_is_mounted; then + ok "$ESP_MOUNTPOINT mounted ($(findmnt -n -o SOURCE "$ESP_MOUNTPOINT"))" + return 0 + fi + problem "$ESP_MOUNTPOINT is not mounted" + if ! acting; then + would "mount the ESP at $ESP_MOUNTPOINT" + return 1 + fi + local dev + dev="$(resolve_esp_device)" || { + problem "could not determine the ESP device" + return 1 + } + mkdir -p "$ESP_MOUNTPOINT" + mount -t vfat "$dev" "$ESP_MOUNTPOINT" || { + problem "mounting $dev at $ESP_MOUNTPOINT failed" + return 1 + } + repaired "mounted $dev at $ESP_MOUNTPOINT" + return 0 +} + +# depmod output is what modprobe consults; the same skipped-hook bug that +# leaves the ESP stale also leaves modules.dep missing for the new kernel. +ensure_depmod() { + local kver dep + kver="$1" + dep="$(rpath "/usr/lib/modules/${kver}/modules.dep")" + if [[ -f $dep ]]; then + ok "modules.dep present for $kver" + return 0 + fi + problem "modules.dep missing for $kver" + if ! acting; then + would "run depmod $kver" + return 1 + fi + if is_live; then + depmod "$kver" || { + problem "depmod $kver failed" + return 1 + } + else + depmod -b "${ROOT%/}" "$kver" || { + problem "depmod failed" + return 1 + } + fi + repaired "generated modules.dep for $kver" + return 0 +} + +# The core repair: make the ESP's kernel image match the installed module tree. +sync_kernel_image() { + local kver src dest have + kver="$1" + src="$(rpath "/usr/lib/modules/${kver}/vmlinuz")" + dest="$(rpath "${ESP_MOUNTPOINT}/vmlinuz-linux")" + + [[ -f $src ]] || { + problem "no kernel image at $src" + return 1 + } + + if [[ -f $dest ]]; then + have="$(kernel_image_version "$dest" || true)" + if [[ $have == "$kver" ]]; then + ok "ESP kernel matches installed tree ($kver)" + return 0 + fi + problem "ESP kernel is ${have:-unreadable} but installed tree is $kver" + else + problem "no kernel image on the ESP" + fi + + if ! acting; then + would "install $kver to $dest" + return 1 + fi + install -Dm644 "$src" "$dest" + repaired "installed kernel $kver to the ESP" + return 0 +} + +# Which kernel an initramfs was built for. Prefers lsinitcpio, but falls back +# to scanning the image directly — lsinitcpio is not guaranteed to be present +# in a degraded emergency shell. +initramfs_version() { + local img="$1" v="" + [[ -r $img ]] || return 1 + if command -v lsinitcpio > /dev/null 2>&1; then + v="$(lsinitcpio "$img" 2> /dev/null | + grep -oE 'modules/[0-9]+\.[0-9]+\.[0-9]+-arch[0-9]+-[0-9]+' | + head -1 | cut -d/ -f2 || true)" + fi + if [[ -z $v ]]; then + v="$(grep -aoE 'modules/[0-9]+\.[0-9]+\.[0-9]+-arch[0-9]+-[0-9]+' "$img" 2> /dev/null | + head -1 | cut -d/ -f2 || true)" + fi + [[ -n $v ]] || return 1 + printf '%s' "$v" +} + +# Rebuild every preset so the initramfs matches the kernel we just installed. +rebuild_initramfs() { + local kver img have + kver="$1" + img="$(rpath "${ESP_MOUNTPOINT}/initramfs-linux.img")" + + have="$(initramfs_version "$img" || true)" + + if [[ -n $have && $have == "$kver" ]]; then + ok "initramfs matches $kver" + return 0 + fi + + problem "initramfs is ${have:-missing/unknown}, expected $kver" + if ! acting; then + would "run mkinitcpio -P" + return 1 + fi + is_live || { + warn "skipping mkinitcpio (--root given)" + return 1 + } + command -v mkinitcpio > /dev/null 2>&1 || { + problem "mkinitcpio is not installed" + return 1 + } + mkinitcpio -P || { + problem "mkinitcpio -P failed" + return 1 + } + repaired "rebuilt initramfs for $kver" + return 0 +} + +# The actual root cause on this machine. With ParallelDownloads > 1, pacman-git +# segfaults every forked hook child, so mkinitcpio and depmod never run and the +# transaction still reports success. +check_parallel_downloads() { + local conf value + conf="$(rpath /etc/pacman.conf)" + [[ -r $conf ]] || return 0 + value="$(grep -E '^[[:space:]]*ParallelDownloads[[:space:]]*=' "$conf" 2> /dev/null | + tail -1 | sed 's/.*=[[:space:]]*//' | tr -d '[:space:]')" + [[ -n $value ]] || { + ok "ParallelDownloads not set (pacman default is serial)" + return 0 + } + if [[ $value == "1" ]]; then + ok "ParallelDownloads = 1" + return 0 + fi + problem "ParallelDownloads = $value — forked alpm hooks segfault, silently skipping mkinitcpio" + if ! acting; then + would "set ParallelDownloads = 1 in $conf" + return 1 + fi + cp -a "$conf" "${conf}.bak-$(date +%Y%m%d-%H%M%S)" + sed -i -E 's/^([[:space:]]*ParallelDownloads[[:space:]]*=[[:space:]]*).*/\11/' "$conf" + repaired "set ParallelDownloads = 1" + return 0 +} + +# ---------------------------------------------------------------------------- +# Modes +# ---------------------------------------------------------------------------- + +# PreTransaction gate: refuse a kernel upgrade when the ESP is not mounted, +# because that upgrade would write the kernel into the root filesystem instead. +run_preflight() { + if boot_is_mounted; then + printf 'boot-repair: ESP mounted at %s — proceeding.\n' "$ESP_MOUNTPOINT" + exit 0 + fi + cat >&2 << EOF + + ==> boot-repair: ABORTING TRANSACTION + + $ESP_MOUNTPOINT is not mounted. Installing a kernel now would write it into + the root filesystem, leaving the firmware booting a stale image whose + modules this upgrade is about to delete. + + Fix it, then retry: + sudo mount $ESP_MOUNTPOINT + sudo $SCRIPT_NAME + +EOF + exit 1 +} + +run_checks() { + local kver + + ensure_root_writable + + msg "Kernel / ESP consistency" + check_fstab_options || true + clean_shadow_files || true + ensure_vfat || true + mount_esp || true + + if ! kver="$(newest_complete_kernel)"; then + problem "no complete kernel module tree found under $(rpath /usr/lib/modules)" + msg "" + msg " Recover the kernel package first (offline, from cache):" + msg " pacman -U /var/cache/pacman/pkg/linux-*.pkg.tar.zst" + return 1 + fi + info "target kernel: $kver" + + ensure_depmod "$kver" || true + + # Only meaningful once the ESP is actually mounted; otherwise these would + # write into the root filesystem — the very bug being repaired. + if boot_is_mounted || ! is_live; then + sync_kernel_image "$kver" || true + rebuild_initramfs "$kver" || true + else + problem "ESP still unmounted — cannot sync the kernel image" + fi + + msg "" + msg "Upgrade safety" + check_parallel_downloads || true + return 0 +} + +# ---------------------------------------------------------------------------- +# Argument parsing +# ---------------------------------------------------------------------------- + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) + MODE="dry-run" + shift + ;; + --preflight) + MODE="preflight" + shift + ;; + --auto) + MODE="auto" + QUIET=1 + shift + ;; + --root) + [[ $# -ge 2 ]] || die "--root requires a directory" + ROOT="$2" + shift 2 + ;; + --esp) + [[ $# -ge 2 ]] || die "--esp requires a device" + ESP_DEV_OVERRIDE="$2" + shift 2 + ;; + -h | --help) usage ;; + *) die "unknown option: $1 (try --help)" ;; + esac +done + +[[ -d $ROOT ]] || die "root directory does not exist: $ROOT" + +if [[ $MODE == "preflight" ]]; then + run_preflight +fi + +# Root is required to modify the live system, but not to repair a tree handed +# to us via --root (a fixture, or a chroot where we are already root). +if acting && is_live && [[ $EUID -ne 0 ]]; then + die "must run as root (try: sudo $SCRIPT_NAME)" +fi + +[[ $QUIET -eq 1 ]] || { + msg "=== boot-repair (${MODE}${ROOT:+, root=$ROOT}) ===" + msg "" +} + +run_checks || true + +# ---------------------------------------------------------------------------- +# Report +# ---------------------------------------------------------------------------- + +# In --auto (post-transaction hook) stay silent unless something happened. +if [[ $MODE == "auto" ]]; then + if [[ $REPAIRS -gt 0 ]]; then + printf 'boot-repair: repaired %d boot inconsistency/ies after this transaction.\n' "$REPAIRS" + fi + if [[ $PROBLEMS -gt $REPAIRS ]]; then + printf 'boot-repair: UNRESOLVED boot problems remain — do NOT reboot. Run: sudo %s\n' "$SCRIPT_NAME" >&2 + exit 1 + fi + exit 0 +fi + +msg "" +if [[ $PROBLEMS -eq 0 ]]; then + msg "=== consistent: nothing to repair ===" + exit 0 +fi + +if [[ $MODE == "dry-run" ]]; then + msg "=== $PROBLEMS problem(s) found — rerun without --dry-run to repair ===" + exit 1 +fi + +if [[ $REPAIRS -ge $PROBLEMS ]]; then + msg "=== repaired $REPAIRS problem(s) ===" + msg "" + msg "Verify before rebooting:" + msg " findmnt $ESP_MOUNTPOINT" + msg " sudo $SCRIPT_NAME --dry-run" + exit 0 +fi + +msg "=== repaired $REPAIRS of $PROBLEMS problem(s) — UNRESOLVED issues remain ===" +exit 1 diff --git a/linux_configuration/scripts/boot_recovery/hooks/05-boot-mounted-guard.hook b/linux_configuration/scripts/boot_recovery/hooks/05-boot-mounted-guard.hook new file mode 100644 index 00000000..e7aba647 --- /dev/null +++ b/linux_configuration/scripts/boot_recovery/hooks/05-boot-mounted-guard.hook @@ -0,0 +1,20 @@ +# Refuse to install a kernel while the ESP is unmounted. +# +# If /boot is not mounted, the kernel image lands in the ROOT filesystem's +# /boot directory instead of the ESP. The firmware keeps booting the previous +# image, whose module tree this same transaction is about to delete — which +# drops the machine into emergency mode on the next boot. +# +# AbortOnFail makes this a gate, not a warning. + +[Trigger] +Type = Path +Operation = Install +Operation = Upgrade +Target = usr/lib/modules/*/vmlinuz + +[Action] +Description = Verifying the ESP is mounted before installing a kernel... +When = PreTransaction +Exec = /usr/local/sbin/boot-repair --preflight +AbortOnFail diff --git a/linux_configuration/scripts/boot_recovery/hooks/99-boot-autorepair.hook b/linux_configuration/scripts/boot_recovery/hooks/99-boot-autorepair.hook new file mode 100644 index 00000000..14e385a2 --- /dev/null +++ b/linux_configuration/scripts/boot_recovery/hooks/99-boot-autorepair.hook @@ -0,0 +1,22 @@ +# After a kernel transaction, verify the ESP actually received the new kernel +# and repair it if it did not. +# +# This is auto-recovery rather than a warning: if mkinitcpio's own hook was +# skipped (the ParallelDownloads segfault bug), this puts the kernel and +# initramfs on the ESP before the machine is ever rebooted. If it cannot, it +# exits non-zero so the transaction ends loudly instead of silently succeeding. +# +# Note: this hook is itself forked by alpm, so it is subject to the same +# segfault bug it compensates for. That is why boot-repair also enforces +# ParallelDownloads = 1 — belt and braces. + +[Trigger] +Type = Path +Operation = Install +Operation = Upgrade +Target = usr/lib/modules/*/vmlinuz + +[Action] +Description = Verifying the ESP received the new kernel... +When = PostTransaction +Exec = /usr/local/sbin/boot-repair --auto diff --git a/linux_configuration/scripts/boot_recovery/install.sh b/linux_configuration/scripts/boot_recovery/install.sh new file mode 100755 index 00000000..085b3511 --- /dev/null +++ b/linux_configuration/scripts/boot_recovery/install.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# ============================================================================ +# Install boot-repair and its pacman gate hooks. +# ============================================================================ +# +# Installs: +# /usr/local/sbin/boot-repair the repair tool +# /etc/pacman.d/hooks/05-boot-mounted-guard.hook pre-transaction gate +# /etc/pacman.d/hooks/99-boot-autorepair.hook post-transaction fix +# +# boot-repair must live on the ROOT filesystem: by definition the ESP is +# unreachable when it is needed. +# ============================================================================ + +set -euo pipefail + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +readonly SCRIPT_DIR +readonly SBIN_DEST="/usr/local/sbin/boot-repair" +readonly HOOK_DIR="/etc/pacman.d/hooks" + +if [[ $EUID -ne 0 ]]; then + echo "This installer needs root; re-running with sudo..." + exec sudo "$0" "$@" +fi + +echo "=== boot-repair installer ===" + +# 1. Dependencies. Everything used is in `base`, so this is a verification +# rather than an install — but it must fail loudly if something is absent, +# since the tool's whole point is working when the system is broken. +echo "[1/4] Checking required tools..." +missing=() +for tool in findmnt depmod modprobe tar install awk sed grep find sort; do + command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") +done +if [[ ${#missing[@]} -gt 0 ]]; then + echo " Missing: ${missing[*]}" + echo " Installing base utilities..." + pacman -S --needed --noconfirm util-linux kmod tar coreutils gawk sed grep findutils +fi +# mkinitcpio is required to rebuild the initramfs. +if ! command -v mkinitcpio >/dev/null 2>&1; then + echo " Installing mkinitcpio..." + pacman -S --needed --noconfirm mkinitcpio +fi +echo " All required tools present." + +# 2. The script itself. +echo "[2/4] Installing $SBIN_DEST..." +install -Dm755 "$SCRIPT_DIR/boot-repair" "$SBIN_DEST" +echo " Installed." + +# 3. Pacman hooks. +echo "[3/4] Installing pacman hooks into $HOOK_DIR..." +mkdir -p "$HOOK_DIR" +for hook in "$SCRIPT_DIR"/hooks/*.hook; do + install -Dm644 "$hook" "$HOOK_DIR/$(basename "$hook")" + echo " $(basename "$hook")" +done + +# 4. Prove the installed copy runs. A recovery tool that was never executed is +# not installed, it is merely copied. +echo "[4/4] Verifying the installed copy..." +if "$SBIN_DEST" --dry-run; then + echo " boot-repair reports the system is consistent." +else + echo " boot-repair found problems (see above). Repair with: sudo $SBIN_DEST" +fi + +echo "=== Installation complete ===" +echo +echo "Usage:" +echo " sudo boot-repair # repair now" +echo " sudo boot-repair --dry-run # report only" +echo +echo "In emergency mode the root filesystem is already mounted, so:" +echo " $SBIN_DEST" diff --git a/linux_configuration/scripts/boot_recovery/tests/live_test.sh b/linux_configuration/scripts/boot_recovery/tests/live_test.sh new file mode 100755 index 00000000..f73aea58 --- /dev/null +++ b/linux_configuration/scripts/boot_recovery/tests/live_test.sh @@ -0,0 +1,160 @@ +#!/bin/bash + +# ============================================================================ +# Reversible live test for boot-repair. +# +# Fixture tests cannot exercise the parts that matter most — unmounting the +# ESP, the pre-transaction gate, and deleting shadow files off the real root +# filesystem. This does, on the real machine, and restores state afterwards. +# +# It refuses to start unless the system is already consistent, and an EXIT trap +# remounts the ESP no matter how it ends. +# +# Run: sudo ./tests/live_test.sh +# ============================================================================ + +set -uo pipefail + +readonly BOOT_REPAIR="/usr/local/sbin/boot-repair" +readonly ESP="/boot" + +PASS=0 +FAIL=0 +UNMOUNTED=0 + +pass() { + PASS=$((PASS + 1)) + printf ' [PASS] %s\n' "$1" +} +fail() { + FAIL=$((FAIL + 1)) + printf ' [FAIL] %s\n' "$1" + [[ -n ${2:-} ]] && printf ' %s\n' "$2" +} + +# Always put the ESP back, however we exit. +restore() { + local rc=$? + if [[ $UNMOUNTED -eq 1 ]] && ! findmnt -n "$ESP" >/dev/null 2>&1; then + echo + echo "RESTORE: remounting $ESP..." + mount "$ESP" && echo " remounted." || echo " !! REMOUNT FAILED — run: sudo mount $ESP" >&2 + fi + exit $rc +} +trap restore EXIT + +[[ $EUID -eq 0 ]] || { + echo "must run as root: sudo $0" >&2 + exit 2 +} +[[ -x $BOOT_REPAIR ]] || { + echo "$BOOT_REPAIR not installed; run ../install.sh first" >&2 + exit 2 +} + +echo "=== boot-repair live test ===" +echo + +# --- Precondition: refuse to run on an already-broken system ---------------- +echo "PRE: system must be consistent before we perturb it" +if "$BOOT_REPAIR" --dry-run >/dev/null 2>&1; then + pass "system is consistent" +else + echo " ABORT: system is not consistent. Repair it first: sudo $BOOT_REPAIR" >&2 + exit 2 +fi + +ESP_KVER_BEFORE="$(grep -aoE '[0-9]+\.[0-9]+\.[0-9]+-arch[0-9]+-[0-9]+' "$ESP/vmlinuz-linux" | head -1)" +echo " ESP kernel before: $ESP_KVER_BEFORE" + +# --- 1. Pre-transaction gate passes while the ESP is mounted ---------------- +echo +echo "TEST 1: --preflight allows a kernel transaction when the ESP is mounted" +if "$BOOT_REPAIR" --preflight >/dev/null 2>&1; then + pass "gate opens (exit 0)" +else + fail "gate opens (exit 0)" "preflight refused a mounted ESP" +fi + +# --- 2. Unmount and confirm the gate closes --------------------------------- +echo +echo "TEST 2: --preflight BLOCKS a kernel transaction when the ESP is unmounted" +umount "$ESP" || { + echo " could not unmount $ESP (in use?)" >&2 + exit 2 +} +UNMOUNTED=1 +if "$BOOT_REPAIR" --preflight >/dev/null 2>&1; then + fail "gate closes (exit 1)" "preflight allowed an unmounted ESP — pacman would corrupt the boot setup" +else + pass "gate closes (exit 1)" +fi + +# --- 3. Dry-run must not write while the ESP is unmounted ------------------- +echo +echo "TEST 3: --dry-run detects the unmounted ESP and writes nothing" +out="$("$BOOT_REPAIR" --dry-run 2>&1)" +if [[ $out == *"is not mounted"* ]]; then + pass "detects unmounted ESP" +else + fail "detects unmounted ESP" "$out" +fi +if [[ -z "$(find "$ESP" -maxdepth 1 -type f 2>/dev/null)" ]]; then + pass "dry-run wrote nothing into the unmounted mountpoint" +else + fail "dry-run wrote nothing into the unmounted mountpoint" "files appeared under $ESP" +fi + +# --- 4. The dangerous path: shadow files on the root filesystem ------------- +echo +echo "TEST 4: shadow files are deleted, then the real ESP is mounted intact" +printf 'FAKE SHADOW vmlinuz 7.1.5-arch1-2\n' >"$ESP/vmlinuz-linux" +printf 'FAKE SHADOW initramfs modules/7.1.5-arch1-2\n' >"$ESP/initramfs-linux.img" +echo " planted 2 shadow files in the unmounted $ESP" + +out="$("$BOOT_REPAIR" 2>&1)" +while IFS= read -r line; do printf ' | %s\n' "$line"; done <<<"$out" + +if [[ $out == *"shadow file"* ]]; then + pass "shadow files detected and removed" +else + fail "shadow files detected and removed" +fi +if findmnt -n "$ESP" >/dev/null 2>&1; then + UNMOUNTED=0 + pass "ESP mounted by the repair" +else + fail "ESP mounted by the repair" +fi + +# --- 5. The real ESP survived unharmed -------------------------------------- +echo +echo "TEST 5: the real ESP is intact" +for d in EFI loader; do + if [[ -d $ESP/$d ]]; then + pass "$ESP/$d preserved" + else + fail "$ESP/$d preserved" "boot-repair destroyed part of the real ESP!" + fi +done + +ESP_KVER_AFTER="$(grep -aoE '[0-9]+\.[0-9]+\.[0-9]+-arch[0-9]+-[0-9]+' "$ESP/vmlinuz-linux" 2>/dev/null | head -1)" +if [[ $ESP_KVER_AFTER == "$ESP_KVER_BEFORE" ]]; then + pass "ESP kernel unchanged ($ESP_KVER_AFTER)" +else + fail "ESP kernel unchanged" "was $ESP_KVER_BEFORE, now ${ESP_KVER_AFTER:-unreadable}" +fi + +# --- 6. Back to a clean bill of health -------------------------------------- +echo +echo "TEST 6: system is consistent again" +if "$BOOT_REPAIR" --dry-run >/dev/null 2>&1; then + pass "final dry-run reports consistent" +else + fail "final dry-run reports consistent" +fi + +echo +echo "=== $PASS passed, $FAIL failed ===" +[[ $FAIL -eq 0 ]] diff --git a/linux_configuration/scripts/boot_recovery/tests/test_boot_repair.sh b/linux_configuration/scripts/boot_recovery/tests/test_boot_repair.sh new file mode 100755 index 00000000..aca769d6 --- /dev/null +++ b/linux_configuration/scripts/boot_recovery/tests/test_boot_repair.sh @@ -0,0 +1,315 @@ +#!/bin/bash + +# ============================================================================ +# Fixture tests for boot-repair. +# +# Each test builds a throwaway root tree in a temp dir and runs boot-repair +# against it with --root, so nothing on the real machine is touched. This +# exercises detection, repair and the safety guards; it cannot exercise +# mount/modprobe, which need the live system. +# ============================================================================ + +set -uo pipefail + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +readonly SCRIPT_DIR +readonly BOOT_REPAIR="$SCRIPT_DIR/../boot-repair" + +PASS=0 +FAIL=0 +WORKROOT="" + +cleanup() { + [[ -n $WORKROOT && -d $WORKROOT ]] && rm -rf "$WORKROOT" +} +trap cleanup EXIT + +pass() { + PASS=$((PASS + 1)) + printf ' [PASS] %s\n' "$1" +} + +fail() { + FAIL=$((FAIL + 1)) + printf ' [FAIL] %s\n' "$1" + [[ -n ${2:-} ]] && printf ' %s\n' "$2" +} + +# assert_contains +assert_contains() { + if [[ $2 == *"$3"* ]]; then + pass "$1" + else + fail "$1" "expected to find: $3" + fi +} + +# assert_not_contains +assert_not_contains() { + if [[ $2 != *"$3"* ]]; then + pass "$1" + else + fail "$1" "did not expect: $3" + fi +} + +# assert_eq +assert_eq() { + if [[ $2 == "$3" ]]; then + pass "$1" + else + fail "$1" "expected '$3', got '$2'" + fi +} + +# ---------------------------------------------------------------------------- +# Fixture construction +# ---------------------------------------------------------------------------- + +# make_root -> prints the path to a fresh fixture root +# +# Builds a consistent system: one complete kernel tree, an ESP whose kernel +# and initramfs match it, a gating fstab and serial downloads. +make_root() { + local name="$1" + local r="$WORKROOT/$name" + local kver="${2:-7.1.5-arch1-2}" + + mkdir -p "$r/usr/lib/modules/$kver/kernel/fs/fat" + mkdir -p "$r/boot" "$r/etc" "$r/var/cache/pacman/pkg" + + # Fake kernel image carrying the version banner boot-repair greps for. + printf 'Linux version %s (linux@archlinux) fake image\n' "$kver" \ + >"$r/usr/lib/modules/$kver/vmlinuz" + : >"$r/usr/lib/modules/$kver/modules.dep" + : >"$r/usr/lib/modules/$kver/kernel/fs/fat/vfat.ko.zst" + + # ESP contents, consistent with the kernel tree. + cp "$r/usr/lib/modules/$kver/vmlinuz" "$r/boot/vmlinuz-linux" + printf 'usr/lib/modules/%s/kernel/fs/fat/vfat.ko.zst\n' "$kver" \ + >"$r/boot/initramfs-linux.img" + mkdir -p "$r/boot/EFI" "$r/boot/loader" + + printf 'UUID=DEAD-BEEF\t/boot\tvfat\tdefaults\t0 2\n' >"$r/etc/fstab" + printf 'ParallelDownloads = 1\n' >"$r/etc/pacman.conf" + + printf '%s' "$r" +} + +run_repair() { + "$BOOT_REPAIR" --root "$1" "${@:2}" 2>&1 +} + +# ---------------------------------------------------------------------------- +# Tests +# ---------------------------------------------------------------------------- + +test_healthy_system() { + echo "TEST: healthy system reports consistent" + local r out rc + r="$(make_root healthy)" + out="$(run_repair "$r" --dry-run)" + rc=$? + assert_eq "exit status is 0" "$rc" "0" + assert_contains "reports consistent" "$out" "nothing to repair" + assert_contains "picks the right kernel" "$out" "target kernel: 7.1.5-arch1-2" +} + +test_stale_esp_kernel_detected() { + echo "TEST: stale ESP kernel is detected and repaired" + local r out rc + r="$(make_root stale)" + # Simulate the real incident: ESP still holds the previous kernel. + printf 'Linux version 7.0.5-arch1-1 (linux@archlinux) fake image\n' \ + >"$r/boot/vmlinuz-linux" + + out="$(run_repair "$r" --dry-run)" + rc=$? + assert_eq "dry-run exits 1" "$rc" "1" + assert_contains "detects the mismatch" "$out" "ESP kernel is 7.0.5-arch1-1 but installed tree is 7.1.5-arch1-2" + assert_contains "dry-run changes nothing" "$out" "would install" + + # The ESP must be untouched by a dry run. + assert_contains "dry-run left ESP alone" "$(cat "$r/boot/vmlinuz-linux")" "7.0.5-arch1-1" + + out="$(run_repair "$r")" + assert_contains "repairs the kernel image" "$out" "installed kernel 7.1.5-arch1-2 to the ESP" + assert_contains "ESP now has the new kernel" "$(cat "$r/boot/vmlinuz-linux")" "7.1.5-arch1-2" +} + +test_incomplete_tree_ignored() { + echo "TEST: a gutted module tree is not chosen as the target" + local r out + r="$(make_root gutted)" + # The exact wreckage the failed upgrade leaves: metadata only, no kernel/, + # no vmlinuz. It sorts newer, so a naive 'newest' pick would take it. + mkdir -p "$r/usr/lib/modules/9.9.9-arch1-1" + : >"$r/usr/lib/modules/9.9.9-arch1-1/modules.dep" + : >"$r/usr/lib/modules/9.9.9-arch1-1/modules.alias" + + out="$(run_repair "$r" --dry-run)" + assert_contains "ignores the incomplete tree" "$out" "target kernel: 7.1.5-arch1-2" + assert_not_contains "does not pick 9.9.9" "$out" "target kernel: 9.9.9-arch1-1" +} + +test_picks_newest_complete() { + echo "TEST: newest COMPLETE tree wins" + local r out kver + r="$(make_root multi)" + kver="7.2.0-arch1-1" + mkdir -p "$r/usr/lib/modules/$kver/kernel/fs/fat" + printf 'Linux version %s (linux@archlinux) fake image\n' "$kver" \ + >"$r/usr/lib/modules/$kver/vmlinuz" + : >"$r/usr/lib/modules/$kver/modules.dep" + + out="$(run_repair "$r" --dry-run)" + assert_contains "selects the newer complete tree" "$out" "target kernel: 7.2.0-arch1-1" +} + +test_fstab_nofail_fixed() { + echo "TEST: noauto/nofail on /boot is detected and reset" + local r out + r="$(make_root fstab)" + printf 'UUID=DEAD-BEEF\t/boot\tvfat\tdefaults,noauto,nofail\t0 2\n' >"$r/etc/fstab" + + out="$(run_repair "$r" --dry-run)" + assert_contains "detects the silent-failure options" "$out" "noauto/nofail" + + out="$(run_repair "$r")" + assert_contains "resets the options" "$out" "reset to 'defaults'" + assert_contains "fstab now gates" "$(cat "$r/etc/fstab")" "defaults" + assert_not_contains "nofail is gone" "$(cat "$r/etc/fstab")" "nofail" + # A backup must exist so the change is reversible. + if compgen -G "$r/etc/fstab.bak-*" >/dev/null; then + pass "fstab backup written" + else + fail "fstab backup written" + fi +} + +test_shadow_files_removed() { + echo "TEST: shadow kernel files in an unmounted /boot are removed" + local r out + r="$(make_root shadow)" + # An unmounted /boot has no EFI/ or loader/ — that is what makes the files + # below shadows on the root filesystem rather than the real ESP. + rm -rf "${r:?}/boot" + mkdir -p "$r/boot" + printf 'Linux version 7.1.5-arch1-2 (linux@archlinux) fake\n' >"$r/boot/vmlinuz-linux" + printf 'usr/lib/modules/7.1.5-arch1-2/x\n' >"$r/boot/initramfs-linux.img" + printf 'stale\n' >"$r/boot/initramfs-linux-fallback.img" + + out="$(run_repair "$r")" + assert_contains "detects the shadows" "$out" "orphaned kernel file" + assert_contains "removes them" "$out" "removed 3 shadow file" + # The fallback image is not regenerated by a --root run, so its absence + # proves the deletion happened. (vmlinuz-linux legitimately reappears: + # sync_kernel_image reinstalls the target kernel straight afterwards.) + if [[ ! -f $r/boot/initramfs-linux-fallback.img ]]; then + pass "shadow fallback image deleted" + else + fail "shadow fallback image deleted" + fi +} + +test_shadow_guard_refuses_real_esp() { + echo "TEST: SAFETY — never delete from a directory that looks like a real ESP" + local r out + r="$(make_root guarded)" + # EFI/ and loader/ are present (make_root creates them), so even though the + # fixture is "unmounted", boot-repair must refuse to delete anything. + out="$(run_repair "$r")" + assert_contains "refuses to delete" "$out" "refusing to delete anything" + if [[ -f $r/boot/vmlinuz-linux ]]; then + pass "real ESP kernel preserved" + else + fail "real ESP kernel preserved" "boot-repair deleted a real ESP kernel!" + fi +} + +test_parallel_downloads_fixed() { + echo "TEST: ParallelDownloads > 1 is detected and set to 1" + local r out + r="$(make_root paralleldl)" + printf 'ParallelDownloads = 5\n' >"$r/etc/pacman.conf" + + out="$(run_repair "$r" --dry-run)" + assert_contains "detects the hook-segfault trigger" "$out" "ParallelDownloads = 5" + + out="$(run_repair "$r")" + assert_contains "sets it to 1" "$out" "set ParallelDownloads = 1" + assert_contains "config updated" "$(cat "$r/etc/pacman.conf")" "ParallelDownloads = 1" +} + +test_missing_modules_dep() { + echo "TEST: missing modules.dep is reported" + local r out + r="$(make_root nodep)" + rm -f "$r/usr/lib/modules/7.1.5-arch1-2/modules.dep" + + out="$(run_repair "$r" --dry-run)" + assert_contains "detects missing modules.dep" "$out" "modules.dep missing for 7.1.5-arch1-2" +} + +test_no_kernel_at_all() { + echo "TEST: no complete kernel tree gives actionable offline guidance" + local r out rc + r="$(make_root nokernel)" + rm -rf "$r/usr/lib/modules" + mkdir -p "$r/usr/lib/modules" + + out="$(run_repair "$r" --dry-run)" + rc=$? + assert_eq "exits non-zero" "$rc" "1" + assert_contains "explains the offline recovery" "$out" "pacman -U /var/cache/pacman/pkg/linux-" +} + +test_help_and_bad_args() { + echo "TEST: CLI contract" + local out rc + out="$("$BOOT_REPAIR" --help 2>&1)" + rc=$? + assert_eq "--help exits 0" "$rc" "0" + assert_contains "help mentions dry-run" "$out" "--dry-run" + + out="$("$BOOT_REPAIR" --nonsense 2>&1)" + rc=$? + assert_eq "unknown option exits 2" "$rc" "2" +} + +# ---------------------------------------------------------------------------- +# Runner +# ---------------------------------------------------------------------------- + +main() { + WORKROOT="$(mktemp -d)" || { + echo "mktemp failed" >&2 + exit 2 + } + + [[ -x $BOOT_REPAIR ]] || { + echo "boot-repair not executable at $BOOT_REPAIR" >&2 + exit 2 + } + + echo "=== boot-repair fixture tests ===" + echo + + test_healthy_system + test_stale_esp_kernel_detected + test_incomplete_tree_ignored + test_picks_newest_complete + test_fstab_nofail_fixed + test_shadow_files_removed + test_shadow_guard_refuses_real_esp + test_parallel_downloads_fixed + test_missing_modules_dep + test_no_kernel_at_all + test_help_and_bad_args + + echo + echo "=== $PASS passed, $FAIL failed ===" + [[ $FAIL -eq 0 ]] +} + +main "$@"