testsAndMisc/linux_configuration/boot_recovery/boot-repair

#!/bin/bash

# ============================================================================
# boot-repair — offline repair for a stale/unwritten EFI System Partition
# ============================================================================
#
# The failure this fixes (seen 2026-07-30):
#   A kernel upgrade replaced /usr/lib/modules/<old> with <new>, but the new
#   kernel image never reached the ESP. The firmware kept booting the OLD
#   vmlinuz, whose module tree no longer existed — so nothing modular could
#   load after pivot-root. In particular `vfat` was gone, /boot itself could
#   not be mounted, local-fs.target failed and systemd dropped to emergency
#   mode. The upgrade was reported as successful.
#
#   Root cause on this machine: pacman-git with ParallelDownloads > 1 makes
#   every forked alpm hook child segfault, silently skipping the
#   90-mkinitcpio-install and depmod hooks.
#
# Design constraints (deliberate):
#   * SELF-CONTAINED — no sourcing of repo libs. It must run from the systemd
#     emergency shell where only the root filesystem is mounted.
#   * FULLY OFFLINE — never touches the network. Missing kernel modules are
#     recovered from /var/cache/pacman/pkg, nothing else.
#   * Repairs by default. Use --dry-run to inspect without changing anything.
#
# Safety invariants:
#   * Files under /boot are ONLY deleted while /boot is not a mountpoint AND
#     contains no EFI/ or loader/ directory AND the system booted via UEFI.
#     Deleting them after mounting, or on a BIOS box that has no ESP at all,
#     would destroy the real kernel and leave the machine unbootable.
#   * EFI/ and loader/ on the ESP are never written or removed.
#   * The running kernel's module tree is never deleted.
# ============================================================================

set -euo pipefail

# Emergency shells can have a threadbare PATH.
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export PATH

readonly SCRIPT_NAME="${0##*/}"
readonly ESP_MOUNTPOINT="/boot"

# Defaults
ROOT="/"
MODE="fix" # fix | dry-run | preflight | auto
ESP_DEV_OVERRIDE=""
PROBLEMS=0
REPAIRS=0
QUIET=0

# ----------------------------------------------------------------------------
# Output helpers
# ----------------------------------------------------------------------------

msg() { [[ $QUIET -eq 1 ]] || printf '%s\n' "$*"; }
ok() { [[ $QUIET -eq 1 ]] || printf '  [ OK ] %s\n' "$*"; }
info() { [[ $QUIET -eq 1 ]] || printf '  [INFO] %s\n' "$*"; }

# A problem that was found. Counted so the exit status can reflect reality.
problem() {
  PROBLEMS=$((PROBLEMS + 1))
  printf '  [FAIL] %s\n' "$*" >&2
}

warn() { printf '  [WARN] %s\n' "$*" >&2; }

# A repair that was actually carried out.
repaired() {
  REPAIRS=$((REPAIRS + 1))
  printf '  [FIXED] %s\n' "$*"
}

die() {
  printf '%s: error: %s\n' "$SCRIPT_NAME" "$*" >&2
  exit 2
}

# True when we are allowed to modify the system.
acting() { [[ $MODE == "fix" || $MODE == "auto" ]]; }

# Describe an action that dry-run would skip.
would() { msg "  [DRY ] would $*"; }

usage() {
  cat << EOF
Usage: $SCRIPT_NAME [OPTIONS]

Repairs a stale EFI System Partition after a kernel upgrade failed to write
it. Repairs by default; runs fully offline.

Options:
  --dry-run        Report problems, change nothing.
  --preflight      Exit non-zero unless the ESP is mounted. For a pacman
                   PreTransaction hook that gates kernel upgrades.
  --auto           Quiet repair for a pacman PostTransaction hook. Prints
                   only when it acts or fails.
  --root DIR       Operate on an alternate root (testing, or an Arch ISO
                   chroot). Live-only steps are skipped.
  --esp DEVICE     Use DEVICE as the ESP instead of auto-detecting it. Also
                   overrides the BIOS guard: on a system with no
                   /sys/firmware/efi, shadow-file cleanup is refused unless
                   the ESP device is named explicitly here.
  -h, --help       Show this help.

Exit status:
  0  system consistent (or fully repaired)
  1  problems remain / preflight gate failed
  2  usage or environment error
EOF
  exit 0
}

# ----------------------------------------------------------------------------
# Libraries. Sourced, not executed: they need this script's strict mode, its
# $ROOT/$MODE globals and the output helpers defined above.
# ----------------------------------------------------------------------------
LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/lib" && pwd)"
# shellcheck source=lib/discovery.sh
source "$LIB_DIR/discovery.sh"
# shellcheck source=lib/repair_fs.sh
source "$LIB_DIR/repair_fs.sh"
# shellcheck source=lib/repair_kernel.sh
source "$LIB_DIR/repair_kernel.sh"
# shellcheck source=lib/modes.sh
source "$LIB_DIR/modes.sh"

# ----------------------------------------------------------------------------
# Argument parsing
# ----------------------------------------------------------------------------

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run)
      MODE="dry-run"
      shift
      ;;
    --preflight)
      MODE="preflight"
      shift
      ;;
    --auto)
      MODE="auto"
      QUIET=1
      shift
      ;;
    --root)
      [[ $# -ge 2 ]] || die "--root requires a directory"
      ROOT="$2"
      shift 2
      ;;
    --esp)
      [[ $# -ge 2 ]] || die "--esp requires a device"
      ESP_DEV_OVERRIDE="$2"
      shift 2
      ;;
    -h | --help) usage ;;
    *) die "unknown option: $1 (try --help)" ;;
  esac
done

[[ -d $ROOT ]] || die "root directory does not exist: $ROOT"

if [[ $MODE == "preflight" ]]; then
  run_preflight
fi

# Root is required to modify the live system, but not to repair a tree handed
# to us via --root (a fixture, or a chroot where we are already root).
if acting && is_live && [[ $EUID -ne 0 ]]; then
  die "must run as root (try: sudo $SCRIPT_NAME)"
fi

[[ $QUIET -eq 1 ]] || {
  msg "=== boot-repair (${MODE}${ROOT:+, root=$ROOT}) ==="
  msg ""
}

run_checks || true

# ----------------------------------------------------------------------------
# Report
# ----------------------------------------------------------------------------

# In --auto (post-transaction hook) stay silent unless something happened.
if [[ $MODE == "auto" ]]; then
  if [[ $REPAIRS -gt 0 ]]; then
    printf 'boot-repair: repaired %d boot inconsistency/ies after this transaction.\n' "$REPAIRS"
  fi
  if [[ $PROBLEMS -gt $REPAIRS ]]; then
    printf 'boot-repair: UNRESOLVED boot problems remain — do NOT reboot. Run: sudo %s\n' "$SCRIPT_NAME" >&2
    exit 1
  fi
  exit 0
fi

msg ""
if [[ $PROBLEMS -eq 0 ]]; then
  msg "=== consistent: nothing to repair ==="
  exit 0
fi

if [[ $MODE == "dry-run" ]]; then
  msg "=== $PROBLEMS problem(s) found — rerun without --dry-run to repair ==="
  exit 1
fi

if [[ $REPAIRS -ge $PROBLEMS ]]; then
  msg "=== repaired $REPAIRS problem(s) ==="
  msg ""
  msg "Verify before rebooting:"
  msg "    findmnt $ESP_MOUNTPOINT"
  msg "    sudo $SCRIPT_NAME --dry-run"
  exit 0
fi

msg "=== repaired $REPAIRS of $PROBLEMS problem(s) — UNRESOLVED issues remain ==="
exit 1