From 334c0cd9045f98cf0abc090806d9182f05e03ee9 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 29 Aug 2026 18:34:25 +0200 Subject: [PATCH] fix: stop the wireguard firewall from silently cutting every container off `write_nftables_ruleset` emitted a forward chain with `policy drop` and no rules at all. That chain runs at the same netfilter hook as Docker's own DOCKER-USER/DOCKER-FORWARD chains, so it vetoed them: every bridged container lost outbound networking while the host itself stayed online, so nothing looked broken. It cost the Signal bot three silent hours today. signal-cli lost its connection to Signal at 15:32:50 and looped on reconnect for 2.5h, while the bot kept answering /healthz with 200 -- `/v1/groups` still served from local state, so every surface said healthy. `flush ruleset` also deletes Docker's own tables, and dockerd only rebuilds them on its next network event, so restore_docker_rules() now restarts docker right after a successful apply. Verified in vmbox (Arch guest, docker installed, alpine probe): BASELINE=reachable -> OLD=blocked -> NEW=reachable, sshd active throughout. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KbjfWdAyHbCubac7wJKJZW --- .../nftables-docker-forward-2026-08-29.json | 42 +++++++++++++++++++ .../features/lib/wg_firewall.sh | 28 +++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 docs/superpowers/evidence/nftables-docker-forward-2026-08-29.json diff --git a/docs/superpowers/evidence/nftables-docker-forward-2026-08-29.json b/docs/superpowers/evidence/nftables-docker-forward-2026-08-29.json new file mode 100644 index 00000000..d1ce6a3f --- /dev/null +++ b/docs/superpowers/evidence/nftables-docker-forward-2026-08-29.json @@ -0,0 +1,42 @@ +{ + "intent": "Stop the WireGuard/SSH firewall from silently cutting every Docker container off the internet, and rebuild Docker's own nft rules after the ruleset flush that deletes them.", + "scope": [ + "linux_configuration/features/lib/wg_firewall.sh (write_nftables_ruleset, verify_nftables_then_apply)", + "Non-goal: no change to the input chain, the WireGuard port, sshd hardening or the LAN/DNS rules", + "Non-goal: no change to Docker's own iptables management (DOCKER-USER stays untouched)" + ], + "changes": [ + "forward chain keeps policy drop but now accepts established/related plus traffic in or out of docker0 and any br-* bridge, so it stops vetoing Docker's own chains at the same netfilter hook", + "new restore_docker_rules() restarts dockerd after a successful apply, because 'flush ruleset' deletes Docker's tables and dockerd only rebuilds them on its next network event" + ], + "verification": [ + { + "command": "vm new nftfw && vm run nftfw 'bash /tmp/nft_docker_guest_test.sh'", + "result": "pass", + "evidence": "Arch guest, docker installed, alpine wget probe to 1.1.1.1: BASELINE=reachable -> OLD(unpatched ruleset)=blocked -> NEW(patched ruleset)=reachable; sshd active throughout. The OLD leg reproduces the outage this fixes." + }, + { + "command": "nft -c -f ", + "result": "pass", + "evidence": "NFT SYNTAX OK for the file write_nftables_ruleset() generates with ALLOW_WEB/ALLOW_DNS both true." + }, + { + "command": "shellcheck -x --severity=style linux_configuration/features/lib/wg_firewall.sh", + "result": "pass", + "evidence": "no findings" + }, + { + "command": "live host check after applying the same five rules by hand", + "result": "pass", + "evidence": "signal-api container regained DNS and outbound; /v1/devices/+420720355763 went 400 'Closed unexpectedly' -> 200; the Signal bot answered !cat and !dog live at 18:28." + } + ], + "risks": [ + "Containers may now forward to the LAN, which is Docker's own default posture but is a relaxation of a bare default-drop forward chain", + "restore_docker_rules restarts dockerd, so containers without a restart policy stay down until started again" + ], + "rollback": [ + "git revert the commit, then re-run setup_wireguard_ssh.sh (or nft -f /etc/nftables.conf) to reapply the previous ruleset", + "After rollback, expect bridged containers to lose outbound networking again: verify with 'docker run --rm --network bridge alpine wget -q -O /dev/null -T 6 https://1.1.1.1'" + ] +} diff --git a/linux_configuration/features/lib/wg_firewall.sh b/linux_configuration/features/lib/wg_firewall.sh index 7ab65db1..909cd879 100755 --- a/linux_configuration/features/lib/wg_firewall.sh +++ b/linux_configuration/features/lib/wg_firewall.sh @@ -50,6 +50,19 @@ table inet filter { } chain forward { type filter hook forward priority 0; policy drop; + + # Docker keeps its own FORWARD rules (DOCKER-USER, DOCKER-FORWARD, + # the isolation chains) in the ip/filter table. This chain runs at + # the same hook, so a bare 'policy drop' here VETOES them and every + # bridged container loses all outbound networking -- while the host + # itself stays perfectly online, so nothing looks broken. That is + # what killed signal-cli's connection to Signal on 2026-08-29: the + # bot stayed up, healthy and deaf for three hours. + ct state established,related accept + iifname "docker0" accept + oifname "docker0" accept + iifname "br-*" accept + oifname "br-*" accept } chain output { type filter hook output priority 0; policy accept; @@ -58,6 +71,20 @@ table inet filter { EOF } +# 'flush ruleset' above deletes Docker's own tables along with everything +# else, and dockerd only rebuilds them on its next network event -- so +# without this, containers keep running with no NAT and no forwarding until +# somebody happens to restart one. Restarting dockerd rebuilds them at once; +# containers with a restart policy come back by themselves. +restore_docker_rules() { + if ! is_service_active docker; then + return 0 + fi + log_warn "Restarting docker so it can rebuild the nft rules the flush removed." + systemctl restart docker + log_ok "docker restarted; its firewall rules are back." +} + verify_nftables_then_apply() { nft -c -f "${NFT_CONF}.new" || die "nftables ruleset failed syntax check -- not applying." mv "${NFT_CONF}.new" "$NFT_CONF" @@ -69,6 +96,7 @@ verify_nftables_then_apply() { die "sshd died after applying nftables -- rolled back. Investigate before retrying." fi log_ok "nftables applied; sshd is still active." + restore_docker_rules log_warn "Before closing this terminal, open a SECOND ssh session now and confirm it connects." enable_service nftables }