testsAndMisc/linux_configuration/scripts/boot_recovery/tests/test_boot_repair.sh
Krzysztof Rudnicki 8e24ff093f Add boot-repair: offline recovery for a stale EFI System Partition
On 2026-07-30 a kernel upgrade left the machine in systemd emergency mode
on three consecutive boots. The upgrade replaced /usr/lib/modules/<old>
with <new>, but the new kernel never reached the ESP, so the firmware kept
booting the OLD vmlinuz whose module tree had just been deleted. Nothing
modular could load after pivot-root — including vfat, which meant /boot
itself could not be mounted, local-fs.target failed, and systemd dropped to
an emergency shell. pacman reported the transaction as successful.

Root cause: pacman-git with ParallelDownloads > 1 segfaults every forked
alpm hook child, silently skipping 90-mkinitcpio-install and depmod.

boot-repair is self-contained (no repo libs — it must run in the emergency
shell) and fully offline (missing modules are recovered from
/var/cache/pacman/pkg). It repairs by default; --dry-run inspects.

It detects and fixes: read-only root, noauto/nofail fstab drift, orphaned
kernel files shadowing an unmounted ESP, unloadable vfat, unmounted ESP,
missing modules.dep, ESP kernel != installed tree, stale initramfs, and
ParallelDownloads > 1.

Two pacman hooks make the failure impossible rather than merely
recoverable: 05-boot-mounted-guard aborts a kernel transaction when the
ESP is unmounted (AbortOnFail), and 99-boot-autorepair fixes ESP drift
after the transaction, before the machine is ever rebooted.

Safety: a module tree counts as usable only if it has BOTH vmlinuz and
kernel/, so the metadata-only wreckage this bug leaves — which sorts
newer — is never chosen as the target. Files under /boot are deleted only
while it is not a mountpoint AND contains no EFI/ or loader/; deleting
after mounting would destroy the real kernel.

Verified:
- 31/31 fixture tests (--root against throwaway trees, no privileges)
- 11/11 reversible live tests on the real machine: unmounted the real ESP,
  planted shadow files, confirmed they were deleted and the ESP then
  mounted with EFI/, loader/ and kernel 7.1.5-arch1-2 intact
- Gate proven against a real transaction with the ESP unmounted:
  "failed to commit transaction", no packages upgraded, nothing written
- shellcheck + shfmt clean, pre-commit passing

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-30 12:48:56 +02:00

316 lines
9.6 KiB
Bash
Executable File

#!/bin/bash
# ============================================================================
# Fixture tests for boot-repair.
#
# Each test builds a throwaway root tree in a temp dir and runs boot-repair
# against it with --root, so nothing on the real machine is touched. This
# exercises detection, repair and the safety guards; it cannot exercise
# mount/modprobe, which need the live system.
# ============================================================================
set -uo pipefail
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
readonly SCRIPT_DIR
readonly BOOT_REPAIR="$SCRIPT_DIR/../boot-repair"
PASS=0
FAIL=0
WORKROOT=""
cleanup() {
[[ -n $WORKROOT && -d $WORKROOT ]] && rm -rf "$WORKROOT"
}
trap cleanup EXIT
pass() {
PASS=$((PASS + 1))
printf ' [PASS] %s\n' "$1"
}
fail() {
FAIL=$((FAIL + 1))
printf ' [FAIL] %s\n' "$1"
[[ -n ${2:-} ]] && printf ' %s\n' "$2"
}
# assert_contains <description> <haystack> <needle>
assert_contains() {
if [[ $2 == *"$3"* ]]; then
pass "$1"
else
fail "$1" "expected to find: $3"
fi
}
# assert_not_contains <description> <haystack> <needle>
assert_not_contains() {
if [[ $2 != *"$3"* ]]; then
pass "$1"
else
fail "$1" "did not expect: $3"
fi
}
# assert_eq <description> <actual> <expected>
assert_eq() {
if [[ $2 == "$3" ]]; then
pass "$1"
else
fail "$1" "expected '$3', got '$2'"
fi
}
# ----------------------------------------------------------------------------
# Fixture construction
# ----------------------------------------------------------------------------
# make_root <name> -> prints the path to a fresh fixture root
#
# Builds a consistent system: one complete kernel tree, an ESP whose kernel
# and initramfs match it, a gating fstab and serial downloads.
make_root() {
local name="$1"
local r="$WORKROOT/$name"
local kver="${2:-7.1.5-arch1-2}"
mkdir -p "$r/usr/lib/modules/$kver/kernel/fs/fat"
mkdir -p "$r/boot" "$r/etc" "$r/var/cache/pacman/pkg"
# Fake kernel image carrying the version banner boot-repair greps for.
printf 'Linux version %s (linux@archlinux) fake image\n' "$kver" \
>"$r/usr/lib/modules/$kver/vmlinuz"
: >"$r/usr/lib/modules/$kver/modules.dep"
: >"$r/usr/lib/modules/$kver/kernel/fs/fat/vfat.ko.zst"
# ESP contents, consistent with the kernel tree.
cp "$r/usr/lib/modules/$kver/vmlinuz" "$r/boot/vmlinuz-linux"
printf 'usr/lib/modules/%s/kernel/fs/fat/vfat.ko.zst\n' "$kver" \
>"$r/boot/initramfs-linux.img"
mkdir -p "$r/boot/EFI" "$r/boot/loader"
printf 'UUID=DEAD-BEEF\t/boot\tvfat\tdefaults\t0 2\n' >"$r/etc/fstab"
printf 'ParallelDownloads = 1\n' >"$r/etc/pacman.conf"
printf '%s' "$r"
}
run_repair() {
"$BOOT_REPAIR" --root "$1" "${@:2}" 2>&1
}
# ----------------------------------------------------------------------------
# Tests
# ----------------------------------------------------------------------------
test_healthy_system() {
echo "TEST: healthy system reports consistent"
local r out rc
r="$(make_root healthy)"
out="$(run_repair "$r" --dry-run)"
rc=$?
assert_eq "exit status is 0" "$rc" "0"
assert_contains "reports consistent" "$out" "nothing to repair"
assert_contains "picks the right kernel" "$out" "target kernel: 7.1.5-arch1-2"
}
test_stale_esp_kernel_detected() {
echo "TEST: stale ESP kernel is detected and repaired"
local r out rc
r="$(make_root stale)"
# Simulate the real incident: ESP still holds the previous kernel.
printf 'Linux version 7.0.5-arch1-1 (linux@archlinux) fake image\n' \
>"$r/boot/vmlinuz-linux"
out="$(run_repair "$r" --dry-run)"
rc=$?
assert_eq "dry-run exits 1" "$rc" "1"
assert_contains "detects the mismatch" "$out" "ESP kernel is 7.0.5-arch1-1 but installed tree is 7.1.5-arch1-2"
assert_contains "dry-run changes nothing" "$out" "would install"
# The ESP must be untouched by a dry run.
assert_contains "dry-run left ESP alone" "$(cat "$r/boot/vmlinuz-linux")" "7.0.5-arch1-1"
out="$(run_repair "$r")"
assert_contains "repairs the kernel image" "$out" "installed kernel 7.1.5-arch1-2 to the ESP"
assert_contains "ESP now has the new kernel" "$(cat "$r/boot/vmlinuz-linux")" "7.1.5-arch1-2"
}
test_incomplete_tree_ignored() {
echo "TEST: a gutted module tree is not chosen as the target"
local r out
r="$(make_root gutted)"
# The exact wreckage the failed upgrade leaves: metadata only, no kernel/,
# no vmlinuz. It sorts newer, so a naive 'newest' pick would take it.
mkdir -p "$r/usr/lib/modules/9.9.9-arch1-1"
: >"$r/usr/lib/modules/9.9.9-arch1-1/modules.dep"
: >"$r/usr/lib/modules/9.9.9-arch1-1/modules.alias"
out="$(run_repair "$r" --dry-run)"
assert_contains "ignores the incomplete tree" "$out" "target kernel: 7.1.5-arch1-2"
assert_not_contains "does not pick 9.9.9" "$out" "target kernel: 9.9.9-arch1-1"
}
test_picks_newest_complete() {
echo "TEST: newest COMPLETE tree wins"
local r out kver
r="$(make_root multi)"
kver="7.2.0-arch1-1"
mkdir -p "$r/usr/lib/modules/$kver/kernel/fs/fat"
printf 'Linux version %s (linux@archlinux) fake image\n' "$kver" \
>"$r/usr/lib/modules/$kver/vmlinuz"
: >"$r/usr/lib/modules/$kver/modules.dep"
out="$(run_repair "$r" --dry-run)"
assert_contains "selects the newer complete tree" "$out" "target kernel: 7.2.0-arch1-1"
}
test_fstab_nofail_fixed() {
echo "TEST: noauto/nofail on /boot is detected and reset"
local r out
r="$(make_root fstab)"
printf 'UUID=DEAD-BEEF\t/boot\tvfat\tdefaults,noauto,nofail\t0 2\n' >"$r/etc/fstab"
out="$(run_repair "$r" --dry-run)"
assert_contains "detects the silent-failure options" "$out" "noauto/nofail"
out="$(run_repair "$r")"
assert_contains "resets the options" "$out" "reset to 'defaults'"
assert_contains "fstab now gates" "$(cat "$r/etc/fstab")" "defaults"
assert_not_contains "nofail is gone" "$(cat "$r/etc/fstab")" "nofail"
# A backup must exist so the change is reversible.
if compgen -G "$r/etc/fstab.bak-*" >/dev/null; then
pass "fstab backup written"
else
fail "fstab backup written"
fi
}
test_shadow_files_removed() {
echo "TEST: shadow kernel files in an unmounted /boot are removed"
local r out
r="$(make_root shadow)"
# An unmounted /boot has no EFI/ or loader/ — that is what makes the files
# below shadows on the root filesystem rather than the real ESP.
rm -rf "${r:?}/boot"
mkdir -p "$r/boot"
printf 'Linux version 7.1.5-arch1-2 (linux@archlinux) fake\n' >"$r/boot/vmlinuz-linux"
printf 'usr/lib/modules/7.1.5-arch1-2/x\n' >"$r/boot/initramfs-linux.img"
printf 'stale\n' >"$r/boot/initramfs-linux-fallback.img"
out="$(run_repair "$r")"
assert_contains "detects the shadows" "$out" "orphaned kernel file"
assert_contains "removes them" "$out" "removed 3 shadow file"
# The fallback image is not regenerated by a --root run, so its absence
# proves the deletion happened. (vmlinuz-linux legitimately reappears:
# sync_kernel_image reinstalls the target kernel straight afterwards.)
if [[ ! -f $r/boot/initramfs-linux-fallback.img ]]; then
pass "shadow fallback image deleted"
else
fail "shadow fallback image deleted"
fi
}
test_shadow_guard_refuses_real_esp() {
echo "TEST: SAFETY — never delete from a directory that looks like a real ESP"
local r out
r="$(make_root guarded)"
# EFI/ and loader/ are present (make_root creates them), so even though the
# fixture is "unmounted", boot-repair must refuse to delete anything.
out="$(run_repair "$r")"
assert_contains "refuses to delete" "$out" "refusing to delete anything"
if [[ -f $r/boot/vmlinuz-linux ]]; then
pass "real ESP kernel preserved"
else
fail "real ESP kernel preserved" "boot-repair deleted a real ESP kernel!"
fi
}
test_parallel_downloads_fixed() {
echo "TEST: ParallelDownloads > 1 is detected and set to 1"
local r out
r="$(make_root paralleldl)"
printf 'ParallelDownloads = 5\n' >"$r/etc/pacman.conf"
out="$(run_repair "$r" --dry-run)"
assert_contains "detects the hook-segfault trigger" "$out" "ParallelDownloads = 5"
out="$(run_repair "$r")"
assert_contains "sets it to 1" "$out" "set ParallelDownloads = 1"
assert_contains "config updated" "$(cat "$r/etc/pacman.conf")" "ParallelDownloads = 1"
}
test_missing_modules_dep() {
echo "TEST: missing modules.dep is reported"
local r out
r="$(make_root nodep)"
rm -f "$r/usr/lib/modules/7.1.5-arch1-2/modules.dep"
out="$(run_repair "$r" --dry-run)"
assert_contains "detects missing modules.dep" "$out" "modules.dep missing for 7.1.5-arch1-2"
}
test_no_kernel_at_all() {
echo "TEST: no complete kernel tree gives actionable offline guidance"
local r out rc
r="$(make_root nokernel)"
rm -rf "$r/usr/lib/modules"
mkdir -p "$r/usr/lib/modules"
out="$(run_repair "$r" --dry-run)"
rc=$?
assert_eq "exits non-zero" "$rc" "1"
assert_contains "explains the offline recovery" "$out" "pacman -U /var/cache/pacman/pkg/linux-"
}
test_help_and_bad_args() {
echo "TEST: CLI contract"
local out rc
out="$("$BOOT_REPAIR" --help 2>&1)"
rc=$?
assert_eq "--help exits 0" "$rc" "0"
assert_contains "help mentions dry-run" "$out" "--dry-run"
out="$("$BOOT_REPAIR" --nonsense 2>&1)"
rc=$?
assert_eq "unknown option exits 2" "$rc" "2"
}
# ----------------------------------------------------------------------------
# Runner
# ----------------------------------------------------------------------------
main() {
WORKROOT="$(mktemp -d)" || {
echo "mktemp failed" >&2
exit 2
}
[[ -x $BOOT_REPAIR ]] || {
echo "boot-repair not executable at $BOOT_REPAIR" >&2
exit 2
}
echo "=== boot-repair fixture tests ==="
echo
test_healthy_system
test_stale_esp_kernel_detected
test_incomplete_tree_ignored
test_picks_newest_complete
test_fstab_nofail_fixed
test_shadow_files_removed
test_shadow_guard_refuses_real_esp
test_parallel_downloads_fixed
test_missing_modules_dep
test_no_kernel_at_all
test_help_and_bad_args
echo
echo "=== $PASS passed, $FAIL failed ==="
[[ $FAIL -eq 0 ]]
}
main "$@"