diff --git a/docs/superpowers/contracts/nftables-drift-self-check-2026-08-30.json b/docs/superpowers/contracts/nftables-drift-self-check-2026-08-30.json new file mode 100644 index 00000000..aa617c03 --- /dev/null +++ b/docs/superpowers/contracts/nftables-drift-self-check-2026-08-30.json @@ -0,0 +1,17 @@ +{ + "title": "Make the nftables drift check run itself", + "objective": "setup_wireguard_ssh.sh verify already gates on the two invariants whose breach caused the 2026-08-29 and 2026-08-30 container-network outages, but nothing ever invoked it, so it reported drift only when a human thought to ask. Install systemd units that run it at every boot and daily, and make a breach a failed unit visible in `systemctl --failed` rather than a log line nobody reads.", + "acceptance_criteria": [ + "nftables-drift.service runs `setup_wireguard_ssh.sh verify` and is enabled at multi-user.target, ordered after nftables.service, docker.service and network-online.target.", + "With /etc/nftables.conf drifted, a reboot leaves the unit in `failed` and it appears in `systemctl --failed`, naming the diff and the missing docker0 accept.", + "With /etc/nftables.conf matching, a reboot leaves the unit active with both invariants OK and no false positive.", + "nftables-drift.timer is armed with Persistent=true so a machine that was off still re-checks.", + "The units are installed by `setup`, by `allow-web`/`allow-dns`, and by a new `install-verify` subcommand, so no step depends on a human remembering." + ], + "out_of_scope": [ + "Any change to the ruleset itself or to verify_nft's two invariants.", + "Notifications: a failed unit is the gate; notify-send from a root system unit is not.", + "Section 1 of the handoff (signal-bot boot resilience), which shipped separately in kuhyx/signal-bot." + ], + "verifier": "linux_configuration/features/lib/tests/test_wg_verify_units.sh under meta/scripts/shell_coverage_jail.sh; plus two real reboots in ~/utils/vmbox (drifted -> failed unit, clean -> active unit) and a live run on the host." +} diff --git a/docs/superpowers/evidence/nftables-drift-self-check-2026-08-30.json b/docs/superpowers/evidence/nftables-drift-self-check-2026-08-30.json new file mode 100644 index 00000000..82dacdac --- /dev/null +++ b/docs/superpowers/evidence/nftables-drift-self-check-2026-08-30.json @@ -0,0 +1,73 @@ +{ + "intent": "A drift check that nobody runs is not a check. Install systemd units that run `setup_wireguard_ssh.sh verify` at every boot and daily, so a stale /etc/nftables.conf fails a unit instead of waiting to be noticed after every bridged container has silently lost the internet.", + "scope": [ + "linux_configuration/features/lib/wg_verify_units.sh", + "linux_configuration/features/setup_wireguard_ssh.sh", + "linux_configuration/features/lib/wg_sshd.sh", + "linux_configuration/features/lib/tests/test_wg_verify_units.sh", + "Non-goal: changing the ruleset or the two invariants verify_nft already gates on", + "Non-goal: notifications; a failed unit in `systemctl --failed` is the gate", + "meta/requirements.txt (unrelated baseline fix, see changes)" + ], + "changes": [ + "New lib wg_verify_units.sh splits rendering from installing, the same way render/write_nftables_ruleset already do, so the generated unit CONTENT is assertable without a daemon-reload.", + "nftables-drift.service is Type=oneshot, After=nftables.service docker.service network-online.target. network-online.target is load-bearing: detect_lan_subnet shells out to `ip route get 1.1.1.1` and dies when there is no route, which at boot would be indistinguishable from real drift.", + "Deliberately NOT Requires=nftables.service: a dead nftables.service means the ruleset is not loaded at all, the most serious version of the fault, and the unit must run and fail rather than be skipped as unsatisfiable.", + "ProtectSystem=strict + ProtectHome=read-only: verify only reads, and the saved LAN_SUBNET lives in the repo under $HOME, where a root-owned rewrite would lock the non-root subcommands out of their own config. The unit makes that write impossible instead of merely unlikely.", + "ExecStart points at the repo checkout, not a copy under /usr/local/bin: the check is 'does /etc match what THIS repo generates', and a stale copy of a drift detector agrees with the drift.", + "The units are installed by `setup`, by `allow-web` and `allow-dns` (the commands that regenerate /etc/nftables.conf, so exactly when the check must exist), and by a new `install-verify` subcommand for retrofitting an existing machine.", + "Bumped the gatelock pin gattag v0.7.2 -> v0.7.3 in meta/requirements.txt to clear a pre-existing red dependency-freshness baseline on main, unrelated to this change but blocking the commit via the ci-baseline-green hook." + ], + "verification": [ + { + "command": "linux_configuration/features/lib/tests/test_wg_verify_units.sh", + "result": "pass", + "evidence": "12 passed, 0 failed. Asserts ExecStart, Type=oneshot, all three After= orderings, the absence of Requires=, both Protect* settings, WantedBy=multi-user.target, and the timer's Unit=/Persistent=/WantedBy=." + }, + { + "command": "meta/scripts/shell_coverage_jail.sh --subject linux_configuration/features/lib/tests/run_all.sh $(cat .../jail_args) --min 0 --fail-on-case-error -- ''", + "result": "pass", + "evidence": "The whole features/lib suite passes under the jail with the new file added; exit 0, no FAIL lines. Same invocation meta/scripts/ci_mirror_shell.sh uses pre-push." + }, + { + "command": "vm run nftdrift 'sudo setup_wireguard_ssh.sh install-verify' then a reboot with /etc/nftables.conf drifted", + "result": "pass", + "evidence": "Boot-time run FAILED as required: `systemctl is-failed` -> failed; `systemctl --failed` lists it; the journal carries the unified diff of the removed `iifname \"docker0\" accept` line AND '[ERROR] live forward chain does NOT accept docker0 -- bridged containers have no network.' Both invariants fired." + }, + { + "command": "vm run nftdrift 'sudo setup_wireguard_ssh.sh allow-web' then a second reboot", + "result": "pass", + "evidence": "No false positive: unit active, journal shows '[ OK ] /etc/nftables.conf matches what this script generates.' and '[ OK ] live forward chain lets Docker through.'; systemctl --failed count for it is 0; timer armed for 00:00." + }, + { + "command": "sudo linux_configuration/features/setup_wireguard_ssh.sh install-verify && sudo systemctl start nftables-drift.service (host)", + "result": "pass", + "evidence": "Installed and enabled on the real machine; the run is green on both invariants and nftables-drift.timer is listed for Mon 2026-08-31 00:00:00 CEST. Not yet observed across a real host reboot -- that was proven in vmbox." + }, + { + "command": "shellcheck -x --severity=style on every changed shell file", + "result": "pass", + "evidence": "clean" + }, + { + "command": "systemd-analyze verify on the rendered units", + "result": "pass", + "evidence": "No findings for either unit; the only output was pre-existing noise about unrelated host units." + }, + { + "command": "git ls-remote --tags https://github.com/kuhyx/utils | grep gatelock", + "result": "pass", + "evidence": "gatelock-v0.7.3 is the newest tag; the pin now names it, which is what the freshness gate compares against." + } + ], + "risks": [ + "ExecStart hardcodes the repo path resolved at install time. Moving or renaming the checkout leaves the unit failing at every boot -- loudly, which is the correct direction, but it is a failure about the unit rather than about the firewall.", + "The host installation has not yet survived a real reboot; only the vmbox guest has. The first real reboot is the true test, and a false positive there would show as a failed unit rather than as a network outage.", + "allow-web/allow-dns now also (re)install the units, so those commands touch /etc/systemd/system where they previously did not." + ], + "rollback": [ + "sudo systemctl disable --now nftables-drift.timer && sudo systemctl disable nftables-drift.service", + "sudo rm /etc/systemd/system/nftables-drift.{service,timer} && sudo systemctl daemon-reload", + "git revert the commit. After rollback: confirm `systemctl list-unit-files nftables-drift.*` reports nothing, and remember that `setup_wireguard_ssh.sh verify` is once again a check nobody runs." + ] +} diff --git a/linux_configuration/features/lib/tests/test_wg_verify_units.sh b/linux_configuration/features/lib/tests/test_wg_verify_units.sh new file mode 100755 index 00000000..199601db --- /dev/null +++ b/linux_configuration/features/lib/tests/test_wg_verify_units.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tests for wg_verify_units.sh — the units that run 'verify' by themselves. +# +# The generated CONTENT is the whole subject. A unit that exists but is +# ordered before nftables.service compares /etc/nftables.conf against the +# previous boot's ruleset; a unit that logs instead of exiting non-zero is a +# warning nobody reads. Both look identical to a presence check, and both +# reproduce the outage this exists to catch. +# +# Rendering writes only where it is told, so unlike its siblings this suite +# needs no jail to be safe -- but it runs inside one with the rest of the +# directory, and must not assume it can reach the real /etc. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./features_harness.sh +. "$SCRIPT_DIR/features_harness.sh" + +_t_setup_env +trap _t_teardown EXIT + +# shellcheck source=../wg_verify_units.sh +. "$SCRIPT_DIR/../wg_verify_units.sh" + +readonly ENTRY="/home/kuhy/testsAndMisc/linux_configuration/features/setup_wireguard_ssh.sh" +UNIT_DIR="$TEST_TMPDIR/units" +mkdir -p "$UNIT_DIR" + +render_verify_units "$UNIT_DIR" "$ENTRY" + +SERVICE="${UNIT_DIR}/${VERIFY_SERVICE}" +TIMER="${UNIT_DIR}/${VERIFY_TIMER}" + +printf '\n-- the service runs the repo checkout, not a copy --\n' +_t_file_has "$SERVICE" "^ExecStart=${ENTRY} verify\$" \ + "ExecStart invokes the entry script's verify subcommand" +_t_file_has "$SERVICE" '^Type=oneshot$' "one shot, not a daemon" + +printf '\n-- ordering: check the ruleset that is actually loaded --\n' +# Before nftables.service, the check reads whatever the previous boot left. +_t_file_has "$SERVICE" '^After=.*nftables\.service' "ordered after nftables.service" +_t_file_has "$SERVICE" '^After=.*docker\.service' "ordered after docker.service" +# detect_lan_subnet shells out to `ip route get` when LAN_SUBNET is unset, and +# dies if there is no route yet -- indistinguishable from real drift. +_t_file_has "$SERVICE" '^After=.*network-online\.target' "ordered after the network is up" +if grep -q '^Requires=' "$SERVICE"; then + _t_fail "Requires= would skip the check when nftables.service is dead, which is the worst fault it detects" +else + _t_pass "no Requires=, so a dead nftables.service still fails this unit" +fi + +printf '\n-- it may not write anything --\n' +_t_file_has "$SERVICE" '^ProtectHome=read-only$' \ + "cannot root-rewrite the saved LAN_SUBNET in the repo" +_t_file_has "$SERVICE" '^ProtectSystem=strict$' "cannot rewrite /etc" + +printf '\n-- it runs at boot, and again on a timer --\n' +_t_file_has "$SERVICE" '^WantedBy=multi-user\.target$' "enabled at boot" +_t_file_has "$TIMER" "^Unit=${VERIFY_SERVICE}\$" "the timer triggers that same service" +_t_file_has "$TIMER" '^Persistent=true$' "a PC that was off still checks when it returns" +_t_file_has "$TIMER" '^WantedBy=timers\.target$' "the timer is enabled at boot too" + +printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" +[[ $FAIL -eq 0 ]] diff --git a/linux_configuration/features/lib/wg_sshd.sh b/linux_configuration/features/lib/wg_sshd.sh index 00645d8a..f857327c 100755 --- a/linux_configuration/features/lib/wg_sshd.sh +++ b/linux_configuration/features/lib/wg_sshd.sh @@ -229,6 +229,8 @@ Commands: allow-dns Open udp/tcp 53 from the LAN in the firewall (for the DNS blocker on this host). verify Check /etc/nftables.conf matches this script and that the live forward chain still lets Docker through. + install-verify Install the units that run 'verify' at every boot, so + drift fails a unit instead of waiting to be noticed. status Show WireGuard/firewall/sshd status. revoke Remove a peer. help Show this message. diff --git a/linux_configuration/features/lib/wg_verify_units.sh b/linux_configuration/features/lib/wg_verify_units.sh new file mode 100755 index 00000000..92818024 --- /dev/null +++ b/linux_configuration/features/lib/wg_verify_units.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# The units that run 'verify' by themselves. +# +# Sourced by setup_wireguard_ssh.sh; split out to keep wg_firewall.sh under +# the 250-line cap. Sourced rather than run, so it inherits the caller's +# strict mode and the variables defined above the source line. +# +# 'verify' gates on two invariants and exits non-zero, but nothing invoked +# it. It reported drift only when a human thought to ask -- and the outage it +# exists for is precisely the one nobody thinks to ask about. Twice now the +# repo has carried the docker0 forward rule while /etc/nftables.conf did not, +# and both times a reboot loaded the stale file and cut every bridged +# container off the internet while the host itself stayed online, so nothing +# looked broken. +# +# So it runs at boot, just after the ruleset is loaded, and a failure lands in +# `systemctl --failed`, where it is visible without being looked for. It must +# FAIL, never warn: a warning in the journal is the same as no check at all. + +readonly VERIFY_SERVICE="nftables-drift.service" +readonly VERIFY_TIMER="nftables-drift.timer" +readonly SYSTEMD_UNIT_DIR="/etc/systemd/system" + +# Rendering is separate from installing so the tests can assert the generated +# content without a daemon-reload, the same split render/write_nftables uses. +# +# The ExecStart deliberately points into the repo checkout rather than a copy +# under /usr/local/bin. The check is "does /etc match what THIS repo +# generates", so a copy would be one more thing that can go stale -- and a +# stale copy of a drift detector agrees with the drift. +render_verify_units() { + local unit_dir="$1" entry="$2" + cat >"${unit_dir}/${VERIFY_SERVICE}" <"${unit_dir}/${VERIFY_TIMER}" </dev/null + enable_service "$VERIFY_TIMER" + log_ok "${VERIFY_SERVICE} will re-check the ruleset at every boot." + log_info "A failure shows up in: systemctl --failed" +} diff --git a/linux_configuration/features/setup_wireguard_ssh.sh b/linux_configuration/features/setup_wireguard_ssh.sh index 992a0ac8..a653b6d0 100755 --- a/linux_configuration/features/setup_wireguard_ssh.sh +++ b/linux_configuration/features/setup_wireguard_ssh.sh @@ -81,6 +81,8 @@ source "$SCRIPT_DIR/lib/wg_keys.sh" source "$SCRIPT_DIR/lib/wg_firewall.sh" # shellcheck source=lib/wg_sshd.sh source "$SCRIPT_DIR/lib/wg_sshd.sh" +# shellcheck source=lib/wg_verify_units.sh +source "$SCRIPT_DIR/lib/wg_verify_units.sh" main() { local cmd="${1:-help}" @@ -89,7 +91,7 @@ main() { # off -- exec sudo "$0" "$@" inside require_root must re-launch with the # subcommand still present, or sudo would silently run with no args. case "$cmd" in - setup | add-peer | revoke | allow-web | allow-dns | verify) + setup | add-peer | revoke | allow-web | allow-dns | verify | install-verify) require_root "$@" ;; esac @@ -103,6 +105,7 @@ main() { enable_wg_service write_nftables_ruleset verify_nftables_then_apply + install_verify_units harden_sshd setup_duckdns print_router_instructions @@ -115,13 +118,22 @@ main() { ;; allow-web) allow_web + # Re-armed on every apply: this is the command that regenerates + # /etc/nftables.conf, so it is exactly when the drift check must + # exist. Leaving it to a separate step people must remember is the + # gap that let the same outage happen twice. + install_verify_units ;; allow-dns) allow_dns + install_verify_units ;; verify) verify_nft ;; + install-verify) + install_verify_units + ;; status) status_cmd ;; diff --git a/meta/requirements.txt b/meta/requirements.txt index 5373af45..1c3a718c 100644 --- a/meta/requirements.txt +++ b/meta/requirements.txt @@ -34,7 +34,7 @@ flake8-pyi==26.5.0 flake8-pytest-style==2.2.0 flake8-return==1.2.0 flake8-simplify==0.30.0 -gatelock @ git+https://github.com/kuhyx/utils@gatelock-v0.7.2#subdirectory=gatelock +gatelock @ git+https://github.com/kuhyx/utils@gatelock-v0.7.3#subdirectory=gatelock genanki==0.13.1 geopandas==1.1.4 howlongtobeatpy==1.0.23