mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-11 22:34:02 +01:00
On 2026-07-30 a kernel upgrade left the machine in systemd emergency mode on three consecutive boots. The upgrade replaced /usr/lib/modules/<old> with <new>, but the new kernel never reached the ESP, so the firmware kept booting the OLD vmlinuz whose module tree had just been deleted. Nothing modular could load after pivot-root — including vfat, which meant /boot itself could not be mounted, local-fs.target failed, and systemd dropped to an emergency shell. pacman reported the transaction as successful. Root cause: pacman-git with ParallelDownloads > 1 segfaults every forked alpm hook child, silently skipping 90-mkinitcpio-install and depmod. boot-repair is self-contained (no repo libs — it must run in the emergency shell) and fully offline (missing modules are recovered from /var/cache/pacman/pkg). It repairs by default; --dry-run inspects. It detects and fixes: read-only root, noauto/nofail fstab drift, orphaned kernel files shadowing an unmounted ESP, unloadable vfat, unmounted ESP, missing modules.dep, ESP kernel != installed tree, stale initramfs, and ParallelDownloads > 1. Two pacman hooks make the failure impossible rather than merely recoverable: 05-boot-mounted-guard aborts a kernel transaction when the ESP is unmounted (AbortOnFail), and 99-boot-autorepair fixes ESP drift after the transaction, before the machine is ever rebooted. Safety: a module tree counts as usable only if it has BOTH vmlinuz and kernel/, so the metadata-only wreckage this bug leaves — which sorts newer — is never chosen as the target. Files under /boot are deleted only while it is not a mountpoint AND contains no EFI/ or loader/; deleting after mounting would destroy the real kernel. Verified: - 31/31 fixture tests (--root against throwaway trees, no privileges) - 11/11 reversible live tests on the real machine: unmounted the real ESP, planted shadow files, confirmed they were deleted and the ESP then mounted with EFI/, loader/ and kernel 7.1.5-arch1-2 intact - Gate proven against a real transaction with the ESP unmounted: "failed to commit transaction", no packages upgraded, nothing written - shellcheck + shfmt clean, pre-commit passing Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
161 lines
4.7 KiB
Bash
Executable File
161 lines
4.7 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# ============================================================================
|
|
# Reversible live test for boot-repair.
|
|
#
|
|
# Fixture tests cannot exercise the parts that matter most — unmounting the
|
|
# ESP, the pre-transaction gate, and deleting shadow files off the real root
|
|
# filesystem. This does, on the real machine, and restores state afterwards.
|
|
#
|
|
# It refuses to start unless the system is already consistent, and an EXIT trap
|
|
# remounts the ESP no matter how it ends.
|
|
#
|
|
# Run: sudo ./tests/live_test.sh
|
|
# ============================================================================
|
|
|
|
set -uo pipefail
|
|
|
|
readonly BOOT_REPAIR="/usr/local/sbin/boot-repair"
|
|
readonly ESP="/boot"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
UNMOUNTED=0
|
|
|
|
pass() {
|
|
PASS=$((PASS + 1))
|
|
printf ' [PASS] %s\n' "$1"
|
|
}
|
|
fail() {
|
|
FAIL=$((FAIL + 1))
|
|
printf ' [FAIL] %s\n' "$1"
|
|
[[ -n ${2:-} ]] && printf ' %s\n' "$2"
|
|
}
|
|
|
|
# Always put the ESP back, however we exit.
|
|
restore() {
|
|
local rc=$?
|
|
if [[ $UNMOUNTED -eq 1 ]] && ! findmnt -n "$ESP" >/dev/null 2>&1; then
|
|
echo
|
|
echo "RESTORE: remounting $ESP..."
|
|
mount "$ESP" && echo " remounted." || echo " !! REMOUNT FAILED — run: sudo mount $ESP" >&2
|
|
fi
|
|
exit $rc
|
|
}
|
|
trap restore EXIT
|
|
|
|
[[ $EUID -eq 0 ]] || {
|
|
echo "must run as root: sudo $0" >&2
|
|
exit 2
|
|
}
|
|
[[ -x $BOOT_REPAIR ]] || {
|
|
echo "$BOOT_REPAIR not installed; run ../install.sh first" >&2
|
|
exit 2
|
|
}
|
|
|
|
echo "=== boot-repair live test ==="
|
|
echo
|
|
|
|
# --- Precondition: refuse to run on an already-broken system ----------------
|
|
echo "PRE: system must be consistent before we perturb it"
|
|
if "$BOOT_REPAIR" --dry-run >/dev/null 2>&1; then
|
|
pass "system is consistent"
|
|
else
|
|
echo " ABORT: system is not consistent. Repair it first: sudo $BOOT_REPAIR" >&2
|
|
exit 2
|
|
fi
|
|
|
|
ESP_KVER_BEFORE="$(grep -aoE '[0-9]+\.[0-9]+\.[0-9]+-arch[0-9]+-[0-9]+' "$ESP/vmlinuz-linux" | head -1)"
|
|
echo " ESP kernel before: $ESP_KVER_BEFORE"
|
|
|
|
# --- 1. Pre-transaction gate passes while the ESP is mounted ----------------
|
|
echo
|
|
echo "TEST 1: --preflight allows a kernel transaction when the ESP is mounted"
|
|
if "$BOOT_REPAIR" --preflight >/dev/null 2>&1; then
|
|
pass "gate opens (exit 0)"
|
|
else
|
|
fail "gate opens (exit 0)" "preflight refused a mounted ESP"
|
|
fi
|
|
|
|
# --- 2. Unmount and confirm the gate closes ---------------------------------
|
|
echo
|
|
echo "TEST 2: --preflight BLOCKS a kernel transaction when the ESP is unmounted"
|
|
umount "$ESP" || {
|
|
echo " could not unmount $ESP (in use?)" >&2
|
|
exit 2
|
|
}
|
|
UNMOUNTED=1
|
|
if "$BOOT_REPAIR" --preflight >/dev/null 2>&1; then
|
|
fail "gate closes (exit 1)" "preflight allowed an unmounted ESP — pacman would corrupt the boot setup"
|
|
else
|
|
pass "gate closes (exit 1)"
|
|
fi
|
|
|
|
# --- 3. Dry-run must not write while the ESP is unmounted -------------------
|
|
echo
|
|
echo "TEST 3: --dry-run detects the unmounted ESP and writes nothing"
|
|
out="$("$BOOT_REPAIR" --dry-run 2>&1)"
|
|
if [[ $out == *"is not mounted"* ]]; then
|
|
pass "detects unmounted ESP"
|
|
else
|
|
fail "detects unmounted ESP" "$out"
|
|
fi
|
|
if [[ -z "$(find "$ESP" -maxdepth 1 -type f 2>/dev/null)" ]]; then
|
|
pass "dry-run wrote nothing into the unmounted mountpoint"
|
|
else
|
|
fail "dry-run wrote nothing into the unmounted mountpoint" "files appeared under $ESP"
|
|
fi
|
|
|
|
# --- 4. The dangerous path: shadow files on the root filesystem -------------
|
|
echo
|
|
echo "TEST 4: shadow files are deleted, then the real ESP is mounted intact"
|
|
printf 'FAKE SHADOW vmlinuz 7.1.5-arch1-2\n' >"$ESP/vmlinuz-linux"
|
|
printf 'FAKE SHADOW initramfs modules/7.1.5-arch1-2\n' >"$ESP/initramfs-linux.img"
|
|
echo " planted 2 shadow files in the unmounted $ESP"
|
|
|
|
out="$("$BOOT_REPAIR" 2>&1)"
|
|
while IFS= read -r line; do printf ' | %s\n' "$line"; done <<<"$out"
|
|
|
|
if [[ $out == *"shadow file"* ]]; then
|
|
pass "shadow files detected and removed"
|
|
else
|
|
fail "shadow files detected and removed"
|
|
fi
|
|
if findmnt -n "$ESP" >/dev/null 2>&1; then
|
|
UNMOUNTED=0
|
|
pass "ESP mounted by the repair"
|
|
else
|
|
fail "ESP mounted by the repair"
|
|
fi
|
|
|
|
# --- 5. The real ESP survived unharmed --------------------------------------
|
|
echo
|
|
echo "TEST 5: the real ESP is intact"
|
|
for d in EFI loader; do
|
|
if [[ -d $ESP/$d ]]; then
|
|
pass "$ESP/$d preserved"
|
|
else
|
|
fail "$ESP/$d preserved" "boot-repair destroyed part of the real ESP!"
|
|
fi
|
|
done
|
|
|
|
ESP_KVER_AFTER="$(grep -aoE '[0-9]+\.[0-9]+\.[0-9]+-arch[0-9]+-[0-9]+' "$ESP/vmlinuz-linux" 2>/dev/null | head -1)"
|
|
if [[ $ESP_KVER_AFTER == "$ESP_KVER_BEFORE" ]]; then
|
|
pass "ESP kernel unchanged ($ESP_KVER_AFTER)"
|
|
else
|
|
fail "ESP kernel unchanged" "was $ESP_KVER_BEFORE, now ${ESP_KVER_AFTER:-unreadable}"
|
|
fi
|
|
|
|
# --- 6. Back to a clean bill of health --------------------------------------
|
|
echo
|
|
echo "TEST 6: system is consistent again"
|
|
if "$BOOT_REPAIR" --dry-run >/dev/null 2>&1; then
|
|
pass "final dry-run reports consistent"
|
|
else
|
|
fail "final dry-run reports consistent"
|
|
fi
|
|
|
|
echo
|
|
echo "=== $PASS passed, $FAIL failed ==="
|
|
[[ $FAIL -eq 0 ]]
|