mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-13 18:30:47 +01:00
On 2026-07-30 a kernel upgrade left the machine in systemd emergency mode on three consecutive boots. The upgrade replaced /usr/lib/modules/<old> with <new>, but the new kernel never reached the ESP, so the firmware kept booting the OLD vmlinuz whose module tree had just been deleted. Nothing modular could load after pivot-root — including vfat, which meant /boot itself could not be mounted, local-fs.target failed, and systemd dropped to an emergency shell. pacman reported the transaction as successful. Root cause: pacman-git with ParallelDownloads > 1 segfaults every forked alpm hook child, silently skipping 90-mkinitcpio-install and depmod. boot-repair is self-contained (no repo libs — it must run in the emergency shell) and fully offline (missing modules are recovered from /var/cache/pacman/pkg). It repairs by default; --dry-run inspects. It detects and fixes: read-only root, noauto/nofail fstab drift, orphaned kernel files shadowing an unmounted ESP, unloadable vfat, unmounted ESP, missing modules.dep, ESP kernel != installed tree, stale initramfs, and ParallelDownloads > 1. Two pacman hooks make the failure impossible rather than merely recoverable: 05-boot-mounted-guard aborts a kernel transaction when the ESP is unmounted (AbortOnFail), and 99-boot-autorepair fixes ESP drift after the transaction, before the machine is ever rebooted. Safety: a module tree counts as usable only if it has BOTH vmlinuz and kernel/, so the metadata-only wreckage this bug leaves — which sorts newer — is never chosen as the target. Files under /boot are deleted only while it is not a mountpoint AND contains no EFI/ or loader/; deleting after mounting would destroy the real kernel. Verified: - 31/31 fixture tests (--root against throwaway trees, no privileges) - 11/11 reversible live tests on the real machine: unmounted the real ESP, planted shadow files, confirmed they were deleted and the ESP then mounted with EFI/, loader/ and kernel 7.1.5-arch1-2 intact - Gate proven against a real transaction with the ESP unmounted: "failed to commit transaction", no packages upgraded, nothing written - shellcheck + shfmt clean, pre-commit passing Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
316 lines
9.6 KiB
Bash
Executable File
316 lines
9.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# ============================================================================
|
|
# Fixture tests for boot-repair.
|
|
#
|
|
# Each test builds a throwaway root tree in a temp dir and runs boot-repair
|
|
# against it with --root, so nothing on the real machine is touched. This
|
|
# exercises detection, repair and the safety guards; it cannot exercise
|
|
# mount/modprobe, which need the live system.
|
|
# ============================================================================
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
|
readonly SCRIPT_DIR
|
|
readonly BOOT_REPAIR="$SCRIPT_DIR/../boot-repair"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
WORKROOT=""
|
|
|
|
cleanup() {
|
|
[[ -n $WORKROOT && -d $WORKROOT ]] && rm -rf "$WORKROOT"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
pass() {
|
|
PASS=$((PASS + 1))
|
|
printf ' [PASS] %s\n' "$1"
|
|
}
|
|
|
|
fail() {
|
|
FAIL=$((FAIL + 1))
|
|
printf ' [FAIL] %s\n' "$1"
|
|
[[ -n ${2:-} ]] && printf ' %s\n' "$2"
|
|
}
|
|
|
|
# assert_contains <description> <haystack> <needle>
|
|
assert_contains() {
|
|
if [[ $2 == *"$3"* ]]; then
|
|
pass "$1"
|
|
else
|
|
fail "$1" "expected to find: $3"
|
|
fi
|
|
}
|
|
|
|
# assert_not_contains <description> <haystack> <needle>
|
|
assert_not_contains() {
|
|
if [[ $2 != *"$3"* ]]; then
|
|
pass "$1"
|
|
else
|
|
fail "$1" "did not expect: $3"
|
|
fi
|
|
}
|
|
|
|
# assert_eq <description> <actual> <expected>
|
|
assert_eq() {
|
|
if [[ $2 == "$3" ]]; then
|
|
pass "$1"
|
|
else
|
|
fail "$1" "expected '$3', got '$2'"
|
|
fi
|
|
}
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Fixture construction
|
|
# ----------------------------------------------------------------------------
|
|
|
|
# make_root <name> -> prints the path to a fresh fixture root
|
|
#
|
|
# Builds a consistent system: one complete kernel tree, an ESP whose kernel
|
|
# and initramfs match it, a gating fstab and serial downloads.
|
|
make_root() {
|
|
local name="$1"
|
|
local r="$WORKROOT/$name"
|
|
local kver="${2:-7.1.5-arch1-2}"
|
|
|
|
mkdir -p "$r/usr/lib/modules/$kver/kernel/fs/fat"
|
|
mkdir -p "$r/boot" "$r/etc" "$r/var/cache/pacman/pkg"
|
|
|
|
# Fake kernel image carrying the version banner boot-repair greps for.
|
|
printf 'Linux version %s (linux@archlinux) fake image\n' "$kver" \
|
|
>"$r/usr/lib/modules/$kver/vmlinuz"
|
|
: >"$r/usr/lib/modules/$kver/modules.dep"
|
|
: >"$r/usr/lib/modules/$kver/kernel/fs/fat/vfat.ko.zst"
|
|
|
|
# ESP contents, consistent with the kernel tree.
|
|
cp "$r/usr/lib/modules/$kver/vmlinuz" "$r/boot/vmlinuz-linux"
|
|
printf 'usr/lib/modules/%s/kernel/fs/fat/vfat.ko.zst\n' "$kver" \
|
|
>"$r/boot/initramfs-linux.img"
|
|
mkdir -p "$r/boot/EFI" "$r/boot/loader"
|
|
|
|
printf 'UUID=DEAD-BEEF\t/boot\tvfat\tdefaults\t0 2\n' >"$r/etc/fstab"
|
|
printf 'ParallelDownloads = 1\n' >"$r/etc/pacman.conf"
|
|
|
|
printf '%s' "$r"
|
|
}
|
|
|
|
run_repair() {
|
|
"$BOOT_REPAIR" --root "$1" "${@:2}" 2>&1
|
|
}
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Tests
|
|
# ----------------------------------------------------------------------------
|
|
|
|
test_healthy_system() {
|
|
echo "TEST: healthy system reports consistent"
|
|
local r out rc
|
|
r="$(make_root healthy)"
|
|
out="$(run_repair "$r" --dry-run)"
|
|
rc=$?
|
|
assert_eq "exit status is 0" "$rc" "0"
|
|
assert_contains "reports consistent" "$out" "nothing to repair"
|
|
assert_contains "picks the right kernel" "$out" "target kernel: 7.1.5-arch1-2"
|
|
}
|
|
|
|
test_stale_esp_kernel_detected() {
|
|
echo "TEST: stale ESP kernel is detected and repaired"
|
|
local r out rc
|
|
r="$(make_root stale)"
|
|
# Simulate the real incident: ESP still holds the previous kernel.
|
|
printf 'Linux version 7.0.5-arch1-1 (linux@archlinux) fake image\n' \
|
|
>"$r/boot/vmlinuz-linux"
|
|
|
|
out="$(run_repair "$r" --dry-run)"
|
|
rc=$?
|
|
assert_eq "dry-run exits 1" "$rc" "1"
|
|
assert_contains "detects the mismatch" "$out" "ESP kernel is 7.0.5-arch1-1 but installed tree is 7.1.5-arch1-2"
|
|
assert_contains "dry-run changes nothing" "$out" "would install"
|
|
|
|
# The ESP must be untouched by a dry run.
|
|
assert_contains "dry-run left ESP alone" "$(cat "$r/boot/vmlinuz-linux")" "7.0.5-arch1-1"
|
|
|
|
out="$(run_repair "$r")"
|
|
assert_contains "repairs the kernel image" "$out" "installed kernel 7.1.5-arch1-2 to the ESP"
|
|
assert_contains "ESP now has the new kernel" "$(cat "$r/boot/vmlinuz-linux")" "7.1.5-arch1-2"
|
|
}
|
|
|
|
test_incomplete_tree_ignored() {
|
|
echo "TEST: a gutted module tree is not chosen as the target"
|
|
local r out
|
|
r="$(make_root gutted)"
|
|
# The exact wreckage the failed upgrade leaves: metadata only, no kernel/,
|
|
# no vmlinuz. It sorts newer, so a naive 'newest' pick would take it.
|
|
mkdir -p "$r/usr/lib/modules/9.9.9-arch1-1"
|
|
: >"$r/usr/lib/modules/9.9.9-arch1-1/modules.dep"
|
|
: >"$r/usr/lib/modules/9.9.9-arch1-1/modules.alias"
|
|
|
|
out="$(run_repair "$r" --dry-run)"
|
|
assert_contains "ignores the incomplete tree" "$out" "target kernel: 7.1.5-arch1-2"
|
|
assert_not_contains "does not pick 9.9.9" "$out" "target kernel: 9.9.9-arch1-1"
|
|
}
|
|
|
|
test_picks_newest_complete() {
|
|
echo "TEST: newest COMPLETE tree wins"
|
|
local r out kver
|
|
r="$(make_root multi)"
|
|
kver="7.2.0-arch1-1"
|
|
mkdir -p "$r/usr/lib/modules/$kver/kernel/fs/fat"
|
|
printf 'Linux version %s (linux@archlinux) fake image\n' "$kver" \
|
|
>"$r/usr/lib/modules/$kver/vmlinuz"
|
|
: >"$r/usr/lib/modules/$kver/modules.dep"
|
|
|
|
out="$(run_repair "$r" --dry-run)"
|
|
assert_contains "selects the newer complete tree" "$out" "target kernel: 7.2.0-arch1-1"
|
|
}
|
|
|
|
test_fstab_nofail_fixed() {
|
|
echo "TEST: noauto/nofail on /boot is detected and reset"
|
|
local r out
|
|
r="$(make_root fstab)"
|
|
printf 'UUID=DEAD-BEEF\t/boot\tvfat\tdefaults,noauto,nofail\t0 2\n' >"$r/etc/fstab"
|
|
|
|
out="$(run_repair "$r" --dry-run)"
|
|
assert_contains "detects the silent-failure options" "$out" "noauto/nofail"
|
|
|
|
out="$(run_repair "$r")"
|
|
assert_contains "resets the options" "$out" "reset to 'defaults'"
|
|
assert_contains "fstab now gates" "$(cat "$r/etc/fstab")" "defaults"
|
|
assert_not_contains "nofail is gone" "$(cat "$r/etc/fstab")" "nofail"
|
|
# A backup must exist so the change is reversible.
|
|
if compgen -G "$r/etc/fstab.bak-*" >/dev/null; then
|
|
pass "fstab backup written"
|
|
else
|
|
fail "fstab backup written"
|
|
fi
|
|
}
|
|
|
|
test_shadow_files_removed() {
|
|
echo "TEST: shadow kernel files in an unmounted /boot are removed"
|
|
local r out
|
|
r="$(make_root shadow)"
|
|
# An unmounted /boot has no EFI/ or loader/ — that is what makes the files
|
|
# below shadows on the root filesystem rather than the real ESP.
|
|
rm -rf "${r:?}/boot"
|
|
mkdir -p "$r/boot"
|
|
printf 'Linux version 7.1.5-arch1-2 (linux@archlinux) fake\n' >"$r/boot/vmlinuz-linux"
|
|
printf 'usr/lib/modules/7.1.5-arch1-2/x\n' >"$r/boot/initramfs-linux.img"
|
|
printf 'stale\n' >"$r/boot/initramfs-linux-fallback.img"
|
|
|
|
out="$(run_repair "$r")"
|
|
assert_contains "detects the shadows" "$out" "orphaned kernel file"
|
|
assert_contains "removes them" "$out" "removed 3 shadow file"
|
|
# The fallback image is not regenerated by a --root run, so its absence
|
|
# proves the deletion happened. (vmlinuz-linux legitimately reappears:
|
|
# sync_kernel_image reinstalls the target kernel straight afterwards.)
|
|
if [[ ! -f $r/boot/initramfs-linux-fallback.img ]]; then
|
|
pass "shadow fallback image deleted"
|
|
else
|
|
fail "shadow fallback image deleted"
|
|
fi
|
|
}
|
|
|
|
test_shadow_guard_refuses_real_esp() {
|
|
echo "TEST: SAFETY — never delete from a directory that looks like a real ESP"
|
|
local r out
|
|
r="$(make_root guarded)"
|
|
# EFI/ and loader/ are present (make_root creates them), so even though the
|
|
# fixture is "unmounted", boot-repair must refuse to delete anything.
|
|
out="$(run_repair "$r")"
|
|
assert_contains "refuses to delete" "$out" "refusing to delete anything"
|
|
if [[ -f $r/boot/vmlinuz-linux ]]; then
|
|
pass "real ESP kernel preserved"
|
|
else
|
|
fail "real ESP kernel preserved" "boot-repair deleted a real ESP kernel!"
|
|
fi
|
|
}
|
|
|
|
test_parallel_downloads_fixed() {
|
|
echo "TEST: ParallelDownloads > 1 is detected and set to 1"
|
|
local r out
|
|
r="$(make_root paralleldl)"
|
|
printf 'ParallelDownloads = 5\n' >"$r/etc/pacman.conf"
|
|
|
|
out="$(run_repair "$r" --dry-run)"
|
|
assert_contains "detects the hook-segfault trigger" "$out" "ParallelDownloads = 5"
|
|
|
|
out="$(run_repair "$r")"
|
|
assert_contains "sets it to 1" "$out" "set ParallelDownloads = 1"
|
|
assert_contains "config updated" "$(cat "$r/etc/pacman.conf")" "ParallelDownloads = 1"
|
|
}
|
|
|
|
test_missing_modules_dep() {
|
|
echo "TEST: missing modules.dep is reported"
|
|
local r out
|
|
r="$(make_root nodep)"
|
|
rm -f "$r/usr/lib/modules/7.1.5-arch1-2/modules.dep"
|
|
|
|
out="$(run_repair "$r" --dry-run)"
|
|
assert_contains "detects missing modules.dep" "$out" "modules.dep missing for 7.1.5-arch1-2"
|
|
}
|
|
|
|
test_no_kernel_at_all() {
|
|
echo "TEST: no complete kernel tree gives actionable offline guidance"
|
|
local r out rc
|
|
r="$(make_root nokernel)"
|
|
rm -rf "$r/usr/lib/modules"
|
|
mkdir -p "$r/usr/lib/modules"
|
|
|
|
out="$(run_repair "$r" --dry-run)"
|
|
rc=$?
|
|
assert_eq "exits non-zero" "$rc" "1"
|
|
assert_contains "explains the offline recovery" "$out" "pacman -U /var/cache/pacman/pkg/linux-"
|
|
}
|
|
|
|
test_help_and_bad_args() {
|
|
echo "TEST: CLI contract"
|
|
local out rc
|
|
out="$("$BOOT_REPAIR" --help 2>&1)"
|
|
rc=$?
|
|
assert_eq "--help exits 0" "$rc" "0"
|
|
assert_contains "help mentions dry-run" "$out" "--dry-run"
|
|
|
|
out="$("$BOOT_REPAIR" --nonsense 2>&1)"
|
|
rc=$?
|
|
assert_eq "unknown option exits 2" "$rc" "2"
|
|
}
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Runner
|
|
# ----------------------------------------------------------------------------
|
|
|
|
main() {
|
|
WORKROOT="$(mktemp -d)" || {
|
|
echo "mktemp failed" >&2
|
|
exit 2
|
|
}
|
|
|
|
[[ -x $BOOT_REPAIR ]] || {
|
|
echo "boot-repair not executable at $BOOT_REPAIR" >&2
|
|
exit 2
|
|
}
|
|
|
|
echo "=== boot-repair fixture tests ==="
|
|
echo
|
|
|
|
test_healthy_system
|
|
test_stale_esp_kernel_detected
|
|
test_incomplete_tree_ignored
|
|
test_picks_newest_complete
|
|
test_fstab_nofail_fixed
|
|
test_shadow_files_removed
|
|
test_shadow_guard_refuses_real_esp
|
|
test_parallel_downloads_fixed
|
|
test_missing_modules_dep
|
|
test_no_kernel_at_all
|
|
test_help_and_bad_args
|
|
|
|
echo
|
|
echo "=== $PASS passed, $FAIL failed ==="
|
|
[[ $FAIL -eq 0 ]]
|
|
}
|
|
|
|
main "$@"
|