diff --git a/docs/superpowers/evidence/loud-maintenance-failures-2026-07-17.json b/docs/superpowers/evidence/loud-maintenance-failures-2026-07-17.json new file mode 100644 index 00000000..acdb2fcf --- /dev/null +++ b/docs/superpowers/evidence/loud-maintenance-failures-2026-07-17.json @@ -0,0 +1,34 @@ +{ + "intent": "Make periodic-maintenance and self-repair failures impossible to miss. Both tools swallowed failures: execute_with_log appended one line to a log nobody reads and returned 0, so the systemd oneshot reported SUCCESS; check_and_enable_services printed one err line among ~100 and exited 0. That combination is why the 2026-05-15 repo reorg silently killed the pacman-wrapper and hosts self-repair for five months with zero visible symptoms — check_hosts detected the disabled hosts guards on every run, failed to find setup_hosts_guard.sh at the stale $CONFIG_DIR/hosts path, and still exited 0.", + "scope": [ + "linux_configuration/scripts/periodic_background/system-maintenance/bin/periodic-system-maintenance.sh (driver template)", + "linux_configuration/scripts/periodic_background/check_and_enable_services.sh", + "Non-goals: enabling the currently-disabled hosts guards (separate decision); the author's in-flight night-lockdown WIP in setup_midnight_shutdown.sh" + ], + "changes": [ + "Driver: added fail_loudly() -> journal at ERROR priority via `logger -p user.err`; collects ALL failures instead of aborting on the first; on any failure writes /var/lib/periodic-system-maintenance/LAST_RUN_FAILED, sends a critical desktop notification (-t 0, never auto-dismisses) to the graphical user via their session bus, and exits 1 so systemd marks the oneshot FAILED (visible in `systemctl --failed` and the timer status) instead of green.", + "Driver: `bash script || rc=$?` so set -e cannot abort the run and every step is still attempted/reported.", + "check_and_enable_services: added err_missing_script() which records + logs at journal ERROR priority; rerouted all 12 'script not found' err() call sites to it; added a terminal banner and `exit 2` when any repair script is missing (a missing repair script means the tool itself is broken).", + "Healthy runs still exit 0 and clear the breadcrumb." + ], + "verification": [ + { + "command": "driver template substituted with nonexistent script paths, temp-redirected LOG_FILE/STATE_DIR, run directly", + "result": "pass", + "evidence": "exit 1; stderr listed all 3 failures; /state/LAST_RUN_FAILED contained all 3; `journalctl -t periodic-system-maintenance -p err` showed 3 FAILURE entries. All four signals fired." + }, + { + "command": "sudo bash setup_periodic_system.sh; sudo /usr/local/bin/periodic-system-maintenance.sh; sudo bash check_and_enable_services.sh --status; shellcheck --severity=warning", + "result": "pass", + "evidence": "live driver regenerated with the loud code; healthy run rc=0 with all 3 steps ✓ and no breadcrumb; checker rc=0 with no banner; shellcheck clean; bash -n clean." + } + ], + "risks": [ + "The hourly oneshot will now genuinely FAIL (red in systemctl --failed) when a repair script is missing or a repair errors — that is the intent, but it makes previously-invisible breakage visible and may surface pre-existing failures.", + "notify_user targets the first graphical session from `loginctl list-sessions`; on a headless boot it silently no-ops (the journal/exit-code/breadcrumb signals still fire)." + ], + "rollback": [ + "git revert the commit and re-run `sudo bash setup_periodic_system.sh` to regenerate the previous (silent) driver.", + "Validate after rollback: /usr/local/bin/periodic-system-maintenance.sh no longer contains fail_loudly; the unit reports success again regardless of failures." + ] +} diff --git a/linux_configuration/scripts/periodic_background/check_and_enable_services.sh b/linux_configuration/scripts/periodic_background/check_and_enable_services.sh index fdd7fc78..a755fad4 100755 --- a/linux_configuration/scripts/periodic_background/check_and_enable_services.sh +++ b/linux_configuration/scripts/periodic_background/check_and_enable_services.sh @@ -57,6 +57,19 @@ warn() { printf "${YELLOW}[!]${NC} %s\n" "$*"; } err() { printf "${RED}[✗]${NC} %s\n" "$*"; } header() { printf "\n${CYAN}=== %s ===${NC}\n" "$*"; } +# A MISSING repair script means THIS TOOL is broken: it silently stops repairing +# the thing it exists to repair. That is exactly how the 2026-05-15 reorg left a +# stale $CONFIG_DIR/hosts path here — check_hosts detected the disabled guards +# every run, failed to find setup_hosts_guard.sh, printed one line among a +# hundred, and still exited 0. Five months of dead self-repair, zero symptoms. +# So: record it, log at journal ERROR priority, and exit non-zero with a banner. +declare -a MISSING_SCRIPTS=() +err_missing_script() { + err "$*" + logger -t check-and-enable-services -p user.err "MISSING REPAIR SCRIPT: $*" + MISSING_SCRIPTS+=("$*") +} + run() { if [[ $DRY_RUN -eq 1 ]]; then echo -e "${YELLOW}DRY-RUN:${NC} $*" @@ -172,7 +185,7 @@ report_and_fix() { _status="ok" fi else - err "Setup script not found: $setup_script" + err_missing_script "Setup script not found: $setup_script" fi fi fi @@ -247,7 +260,7 @@ check_pacman_wrapper() { status="ok" fi else - err "Installer script not found: $PACMAN_WRAPPER_INSTALL" + err_missing_script "Installer script not found: $PACMAN_WRAPPER_INSTALL" fi fi fi @@ -326,7 +339,7 @@ check_makepkg_wrapper() { status="ok" fi else - err "Installer script not found: $MAKEPKG_WRAPPER_INSTALL" + err_missing_script "Installer script not found: $MAKEPKG_WRAPPER_INSTALL" fi fi fi @@ -770,7 +783,7 @@ check_hosts() { run bash "$HOSTS_INSTALL_SCRIPT" ((FIXES_APPLIED++)) || true else - err "Hosts install script not found: $HOSTS_INSTALL_SCRIPT" + err_missing_script "Hosts install script not found: $HOSTS_INSTALL_SCRIPT" fi fi @@ -788,7 +801,7 @@ check_hosts() { run bash "$HOSTS_GUARD_SCRIPT" ((FIXES_APPLIED++)) || true else - err "Hosts guard script not found: $HOSTS_GUARD_SCRIPT" + err_missing_script "Hosts guard script not found: $HOSTS_GUARD_SCRIPT" fi fi @@ -799,7 +812,7 @@ check_hosts() { run bash "$HOSTS_PACMAN_HOOKS_SCRIPT" ((FIXES_APPLIED++)) || true else - err "Pacman hooks script not found: $HOSTS_PACMAN_HOOKS_SCRIPT" + err_missing_script "Pacman hooks script not found: $HOSTS_PACMAN_HOOKS_SCRIPT" fi fi @@ -926,7 +939,7 @@ check_focus_mode() { run sudo -u "$user" bash "$FOCUS_MODE_SCRIPT" install ((FIXES_APPLIED++)) || true else - err "Install script not found: $FOCUS_MODE_SCRIPT" + err_missing_script "Install script not found: $FOCUS_MODE_SCRIPT" fi fi fi @@ -984,7 +997,7 @@ check_compulsive_blocker() { run bash "$COMPULSIVE_BLOCK_SCRIPT" install ((FIXES_APPLIED++)) || true else - err "Install script not found: $COMPULSIVE_BLOCK_SCRIPT" + err_missing_script "Install script not found: $COMPULSIVE_BLOCK_SCRIPT" fi fi fi @@ -1074,7 +1087,7 @@ check_leechblock() { run sudo -u "$user" bash "$LEECHBLOCK_SCRIPT" ((FIXES_APPLIED++)) || true else - err "Install script not found: $LEECHBLOCK_SCRIPT" + err_missing_script "Install script not found: $LEECHBLOCK_SCRIPT" fi fi fi @@ -1126,7 +1139,7 @@ check_guest_mode_removal() { run bash "$REMOVE_GUEST_MODE_SCRIPT" ((FIXES_APPLIED++)) || true else - err "Script not found: $REMOVE_GUEST_MODE_SCRIPT" + err_missing_script "Script not found: $REMOVE_GUEST_MODE_SCRIPT" fi fi fi @@ -1167,7 +1180,7 @@ check_vbox_hosts() { run bash "$VBOX_HOSTS_SCRIPT" ((FIXES_APPLIED++)) || true else - err "Script not found: $VBOX_HOSTS_SCRIPT" + err_missing_script "Script not found: $VBOX_HOSTS_SCRIPT" fi fi fi @@ -1226,7 +1239,7 @@ check_workout_locker() { status="ok" fi else - err "Install script not found: $WORKOUT_LOCKER_INSTALL_SCRIPT" + err_missing_script "Install script not found: $WORKOUT_LOCKER_INSTALL_SCRIPT" fi fi fi @@ -1310,3 +1323,25 @@ main() { } main + +###################################################################### +# A missing repair script means this tool cannot do the one job it exists for. +# Exiting 0 there is indistinguishable from "everything is fine" — which is how +# it hid dead hosts self-repair for five months. Make it impossible to miss. +###################################################################### +if [[ ${#MISSING_SCRIPTS[@]} -gt 0 ]]; then + printf "\n${RED}" + printf '=%.0s' {1..74} + printf "\n SELF-REPAIR IS BROKEN — %d repair script(s) MISSING\n" "${#MISSING_SCRIPTS[@]}" + printf '=%.0s' {1..74} + printf "${NC}\n" + for missing in "${MISSING_SCRIPTS[@]}"; do + err "$missing" + done + printf "${RED}The services above are NOT being repaired — this tool found the\n" + printf "problem and then could not run the fix. Correct these paths first.${NC}\n" + printf "${CYAN}Also logged at error priority: journalctl -p err -t check-and-enable-services${NC}\n\n" + exit 2 +fi + +exit 0 diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/periodic-system-maintenance.sh b/linux_configuration/scripts/periodic_background/system-maintenance/bin/periodic-system-maintenance.sh index 44fb83b1..7570b9f1 100755 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/periodic-system-maintenance.sh +++ b/linux_configuration/scripts/periodic_background/system-maintenance/bin/periodic-system-maintenance.sh @@ -1,54 +1,113 @@ #!/bin/bash # Periodic system maintenance script -# Installs pacman wrapper and updates hosts file +# Installs pacman/makepkg wrappers and updates hosts file # This file is installed by setup_periodic_system.sh +# +# FAILURES ARE LOUD, ON PURPOSE. +# This script used to swallow every failure: a missing repair script was one +# line appended to a log nobody reads, execute_with_log returned 0, and the +# systemd unit therefore reported SUCCESS. That is exactly how a stale path +# (from the 2026-05-15 repo reorg) silently disabled the pacman-wrapper and +# hosts self-repair for five months without a single visible symptom. +# +# Any failure now surfaces four ways: +# 1. journal at ERROR priority -> journalctl -p err -t periodic-system-maintenance +# 2. the systemd unit FAILS -> systemctl --failed / red timer (exit 1 below) +# 3. a critical desktop notification that does not auto-dismiss +# 4. a breadcrumb file -> /var/lib/periodic-system-maintenance/LAST_RUN_FAILED set -e LOG_FILE="/var/log/periodic-system-maintenance.log" +STATE_DIR="/var/lib/periodic-system-maintenance" +FAILURE_FLAG="$STATE_DIR/LAST_RUN_FAILED" # Path placeholders replaced at install time PACMAN_WRAPPER_INSTALL="__PACMAN_WRAPPER_INSTALL__" MAKEPKG_WRAPPER_INSTALL="__MAKEPKG_WRAPPER_INSTALL__" HOSTS_INSTALL_SCRIPT="__HOSTS_INSTALL_SCRIPT__" +declare -a FAILURES=() + # Function to log with timestamp log_message() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" } +# Record a failure loudly: journal at ERROR priority + collect for the summary. +fail_loudly() { + local msg="$1" + log_message "FAILURE: $msg" + echo "✗ $msg" >&2 + logger -t periodic-system-maintenance -p user.err "FAILURE: $msg" + FAILURES+=("$msg") +} + +# Desktop notification for the graphical user (this runs as root under systemd, +# so we have to reach the user's session bus explicitly). +notify_user() { + local title="$1" body="$2" + command -v notify-send >/dev/null 2>&1 || return 0 + local target_user target_uid + target_user="$(loginctl list-sessions --no-legend 2>/dev/null | awk 'NF {print $3; exit}')" + [[ -n $target_user ]] || return 0 + target_uid="$(id -u "$target_user" 2>/dev/null)" || return 0 + # -t 0 => never auto-dismiss; the user must acknowledge it. + sudo -u "$target_user" \ + DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$target_uid/bus" \ + notify-send -u critical -t 0 "$title" "$body" 2>/dev/null || true +} + # Function to execute with logging execute_with_log() { local script_path="$1" local script_name="$2" + local rc=0 log_message "Starting $script_name" echo "Executing $script_name..." >&2 - if [[ -f $script_path ]]; then - if bash "$script_path" >> "$LOG_FILE" 2>&1; then - log_message "$script_name completed successfully" - echo "✓ $script_name completed successfully" >&2 - else - local ec=$? - log_message "$script_name failed with exit code $ec" - echo "✗ $script_name failed (exit $ec)" >&2 - fi + if [[ ! -f $script_path ]]; then + fail_loudly "$script_name: script NOT FOUND at $script_path" + return 0 + fi + + # `|| rc=$?` keeps set -e from aborting so every step still runs and we can + # report ALL failures, not just the first. + bash "$script_path" >> "$LOG_FILE" 2>&1 || rc=$? + if ((rc == 0)); then + log_message "$script_name completed successfully" + echo "✓ $script_name completed successfully" >&2 else - log_message "$script_name not found at $script_path" - echo "✗ $script_name not found at $script_path" >&2 + fail_loudly "$script_name: FAILED (exit $rc) — see $LOG_FILE" fi } # Start maintenance log_message "=== Periodic System Maintenance Started ===" -# Install pacman wrapper +# Install pacman/makepkg wrappers execute_with_log "$PACMAN_WRAPPER_INSTALL" "Pacman Wrapper Installation" execute_with_log "$MAKEPKG_WRAPPER_INSTALL" "Makepkg Wrapper Installation" # Update hosts file execute_with_log "$HOSTS_INSTALL_SCRIPT" "Hosts File Update" +if ((${#FAILURES[@]} > 0)); then + mkdir -p "$STATE_DIR" + printf '%s\n' "${FAILURES[@]}" > "$FAILURE_FLAG" + log_message "=== Periodic System Maintenance FAILED (${#FAILURES[@]} failure(s)) ===" + echo "✗ Periodic system maintenance FAILED with ${#FAILURES[@]} failure(s):" >&2 + printf ' - %s\n' "${FAILURES[@]}" >&2 + notify_user "⚠ System maintenance FAILED" \ + "$(printf '%s\n' "${FAILURES[@]}") + +journalctl -p err -t periodic-system-maintenance" + # Non-zero => systemd marks the unit failed, so `systemctl --failed` and the + # timer's status show it instead of a green "success". + exit 1 +fi + +rm -f "$FAILURE_FLAG" 2>/dev/null || true log_message "=== Periodic System Maintenance Completed ===" echo "Periodic system maintenance completed. Check $LOG_FILE for details." >&2