testsAndMisc/linux_configuration/features/lib/tests/test_dot_resolver_install.sh

#!/usr/bin/env bash
# Tests for dot_resolver_install.sh — the DoT resolver's install half.
#
# Every function here sudo-writes a systemd unit, an stunnel config or an
# nftables rule against LIVE infrastructure (dns.kuhy.duckdns.org:853). They
# are executed for real, not mocked, because the generated CONTENT is the
# thing worth asserting -- a unit file with the wrong ExecStart is exactly the
# bug a presence check would miss.
#
# This is only safe under meta/scripts/shell_coverage_jail.sh, which runs the
# suite inside a user+mount namespace with /etc, /usr/local/bin and /var/lib
# bind-mounted to a throwaway dir. Run directly on a real host it would
# rewrite the resolver, so it refuses to start unless the jail is in place.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./features_harness.sh
. "$SCRIPT_DIR/features_harness.sh"

# --- Refuse to run outside the jail -----------------------------------------
# The jail bind-mounts /etc, so the real resolver's config is absent inside it.
# Seeing the host's own stunnel config means we are NOT contained.
if [[ -f /etc/stunnel/dot-resolver.conf ]] && [[ ! -w /etc ]]; then
	printf 'REFUSING: this suite must run under shell_coverage_jail.sh\n' >&2
	exit 1
fi

_t_setup_env
trap _t_teardown EXIT

# The values the entry script defines above its source line.
SCRIPT_NAME="setup_dot_resolver.sh"
CERT_DIR="/etc/stunnel/dot"
STUNNEL_CONF="/etc/stunnel/dot-resolver.conf"
SYNC_SCRIPT="/usr/local/bin/sync-dot-cert.sh"
DOT_PORT=853
DOMAIN="dns.kuhy.duckdns.org"
BIND_ADDR="10.8.0.1"
CADDY_CONTAINER="caddy"

# install_cert_sync installs this tracked file verbatim (no heredoc): every
# script under /usr/local/bin must be a file in a repo.
SYNC_SCRIPT_SRC="$(cd "$SCRIPT_DIR/../../../scripts" && pwd)/sync-dot-cert.sh"
die() {
	printf 'die: %s\n' "$1" >&2
	return 1
}

# shellcheck source=../dot_resolver_install.sh
. "$FEATURES_LIB_DIR/dot_resolver_install.sh"

# Fixture self-check: a typo in one of these names would turn every assertion
# below into a silent no-op rather than a failure.
[[ -n $CERT_DIR && -n $STUNNEL_CONF && -n $SYNC_SCRIPT ]] ||
	{
		printf 'fixture globals are not populated\n' >&2
		exit 1
	}

mkdir -p /etc/stunnel /etc/systemd/system /usr/local/bin

# --- install_cert_sync ------------------------------------------------------
install_cert_sync
if cmp -s "$SYNC_SCRIPT_SRC" "$SYNC_SCRIPT"; then
	_t_pass "the installed sync script is the tracked file, byte for byte"
else
	_t_fail "the installed sync script is the tracked file, byte for byte"
fi
if grep -q 'Generated by' "$SYNC_SCRIPT"; then
	_t_fail "the sync script is not generated"
else
	_t_pass "the sync script is not generated"
fi
if [[ -d $CERT_DIR ]]; then
	_t_pass "cert dir created"
else
	_t_fail "cert dir created"
fi
_t_eq "700" "$(stat -c '%a' "$CERT_DIR")" "cert dir is mode 700 (it holds a private key)"
_t_eq "755" "$(stat -c '%a' "$SYNC_SCRIPT")" "sync script is executable"
# Reinstall over a stale generated copy: install replaces it (idempotent).
printf '#!/bin/bash\n# Generated by setup_dot_resolver.sh\n' >"$SYNC_SCRIPT"
install_cert_sync
_t_eq "$(sha256sum <"$SYNC_SCRIPT_SRC")" "$(sha256sum <"$SYNC_SCRIPT")" "a stale generated copy is replaced"
out="$(SYNC_SCRIPT_SRC="$TEST_TMPDIR/absent.sh" install_cert_sync 2>&1 || true)"
_t_has "$out" 'tracked sync script missing' "a missing tracked file stops the install"

# --- the tracked sync script itself -----------------------------------------
# docker cp is stubbed to copy from a fixture tree, so the script's own logic
# (assemble, compare, install, reload-only-on-change) runs for real.
SYNC_CERTS="$TEST_TMPDIR/caddy"
mkdir -p "$SYNC_CERTS"
cat >"$TEST_TMPDIR/bin/docker" <<DOCKER
#!/usr/bin/env bash
printf 'docker %s\n' "\$*" >>"$TEST_TMPDIR/calls.log"
cp "$SYNC_CERTS/\$(basename "\${2#*:}")" "\$3"
DOCKER
chmod +x "$TEST_TMPDIR/bin/docker"
_t_stub systemctl
sync_dir="$TEST_TMPDIR/stunnel-dot"
mkdir -p "$sync_dir"
printf 'CERT1\n' >"$SYNC_CERTS/d.example.crt"
printf 'KEY1\n' >"$SYNC_CERTS/d.example.key"
out="$("$SYNC_SCRIPT_SRC" d.example box "$sync_dir")"
_t_eq "$(printf 'CERT1\nKEY1')" "$(cat "$sync_dir/combined.pem")" "the sync assembles cert + key"
_t_eq "600" "$(stat -c '%a' "$sync_dir/combined.pem")" "combined.pem is mode 600"
_t_has "$out" 'certificate updated for d.example' "a new cert is announced"
_t_called 'docker cp box:/data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/d.example/d.example.crt' \
	"the cert is read from the domain's Caddy dir in the named container"
_t_called 'systemctl reload-or-restart stunnel-dot.service' "a new cert reloads stunnel"
: >"$TEST_TMPDIR/calls.log"
out="$("$SYNC_SCRIPT_SRC" d.example box "$sync_dir")"
_t_eq "" "$out" "an unchanged cert is silent"
if grep -q reload "$TEST_TMPDIR/calls.log"; then
	_t_fail "an unchanged cert does not reload stunnel"
else
	_t_pass "an unchanged cert does not reload stunnel"
fi
_t_unstub docker
_t_unstub systemctl

# --- install_stunnel_conf ---------------------------------------------------
install_stunnel_conf
_t_file_has "$STUNNEL_CONF" "accept = $BIND_ADDR:$DOT_PORT" "stunnel accepts on the bind address"
_t_file_has "$STUNNEL_CONF" 'connect = 127.0.0.1:53' "stunnel forwards to local dnsmasq"
_t_file_has "$STUNNEL_CONF" 'sslVersionMin = TLSv1.2' "stunnel refuses pre-TLS1.2 (Android requires it)"
_t_file_has "$STUNNEL_CONF" "cert = $CERT_DIR/combined.pem" "stunnel reads the synced cert"

# --- install_units ----------------------------------------------------------
_t_stub systemctl
install_units
_t_file_has /etc/systemd/system/stunnel-dot.service 'stunnel' "stunnel unit installed"
# Requires= turned one failed dnsmasq start at boot into a DoT outage that
# outlived dnsmasq's own 5 s recovery; Wants= lets stunnel start regardless.
_t_file_has /etc/systemd/system/stunnel-dot.service '^Wants=dnsmasq\.service$' "stunnel only Wants dnsmasq"
if grep -q '^Requires=dnsmasq' /etc/systemd/system/stunnel-dot.service; then
	_t_fail "Requires=dnsmasq.service would fail stunnel whenever dnsmasq's first start fails"
else
	_t_pass "no Requires=dnsmasq.service"
fi
_t_file_has /etc/systemd/system/dot-cert-sync.service "^ExecStart=$SYNC_SCRIPT $DOMAIN $CADDY_CONTAINER $CERT_DIR\$" \
	"cert-sync service passes domain, container and cert dir"
_t_file_has /etc/systemd/system/dot-cert-sync.timer 'OnCalendar' "cert-sync timer is scheduled"
_t_called 'daemon-reload' "units are followed by a daemon-reload"

# --- open_firewall: the WireGuard-only branch -------------------------------
# nft absent entirely: the function must still return cleanly rather than
# aborting the install on a host without nftables.
out="$(open_firewall 2>&1)"
_t_has "$out" 'WireGuard-only' "wg-bound install needs no router forward"

# nft present: the rule goes through the firewall owner's allow-dot, which
# persists ALLOW_DOT so a later re-apply keeps it (a bare 'nft add rule' was
# silently dropped by the next one).
_t_stub nft
WG_SCRIPT="$TEST_TMPDIR/bin/wg_owner"
printf '#!/usr/bin/env bash\nprintf "wg_owner %%s\\n" "$*" >>"%s/calls.log"\nexit 0\n' \
	"$TEST_TMPDIR" >"$WG_SCRIPT"
chmod +x "$WG_SCRIPT"
out="$(open_firewall 2>&1)"
_t_has "$out" 'wg0 only (persisted)' "the wg0 rule is opened through the firewall owner"
_t_called 'wg_owner allow-dot' "open_firewall runs setup_wireguard_ssh.sh allow-dot"
if grep -q 'add rule inet filter input iifname wg0' "$TEST_TMPDIR/calls.log"; then
	_t_fail "no unpersisted runtime 'nft add rule' for the wg0 branch"
else
	_t_pass "no unpersisted runtime 'nft add rule' for the wg0 branch"
fi

# allow-dot fails (e.g. sshd check rolled it back): warn and carry on.
printf '#!/usr/bin/env bash\nexit 1\n' >"$WG_SCRIPT"
out="$(open_firewall 2>&1)"
_t_has "$out" 'WARNING: could not open' "a failed allow-dot warns instead of aborting"
unset WG_SCRIPT

# The public branch still adds its rule at runtime; its add-failure path
# needs an nft whose 'list' succeeds and 'add' fails.
cat >"$TEST_TMPDIR/bin/nft" <<'NFT'
#!/usr/bin/env bash
printf 'nft %s\n' "$*" >>"$TEST_TMPDIR/calls.log"
[[ $* == *add* ]] && exit 1
exit 0
NFT
chmod +x "$TEST_TMPDIR/bin/nft"

# --- open_firewall: the publicly-bound branch -------------------------------
# A public bind is an open resolver, so the rule MUST carry a rate limit.
BIND_ADDR="0.0.0.0"
out="$(open_firewall 2>&1)"
_t_has "$out" 'rate limited' "a public bind is rate limited"
_t_called 'limit rate 25/second burst 50' "the public rule carries the documented limit"
_t_has "$out" 'router forwards' "a public bind asks for a router forward"

# The same add-failure path on the public branch.
out="$(open_firewall 2>&1)"
_t_has "$out" 'WARNING: could not add rate-limited nftables rule' \
	"a failed public rule add warns instead of aborting"
BIND_ADDR="10.8.0.1"

# --- run_check --------------------------------------------------------------
# Everything healthy: stunnel active, port listening, cert present.
printf '#!/usr/bin/env bash\nexit 0\n' >"$TEST_TMPDIR/bin/systemctl"
chmod +x "$TEST_TMPDIR/bin/systemctl"
printf '#!/usr/bin/env bash\nprintf "LISTEN 0 0 10.8.0.1:%s\\n"\n' "$DOT_PORT" >"$TEST_TMPDIR/bin/ss"
chmod +x "$TEST_TMPDIR/bin/ss"
install -m 600 /dev/null "$CERT_DIR/combined.pem"
printf 'cert\n' >"$CERT_DIR/combined.pem"
if run_check >/dev/null 2>&1; then
	_t_pass "run_check passes on a healthy install"
else
	_t_fail "run_check passes on a healthy install"
fi

# stunnel down: must fail and say which check failed.
printf '#!/usr/bin/env bash\n[[ $* == *is-active* ]] && exit 1\nexit 0\n' >"$TEST_TMPDIR/bin/systemctl"
chmod +x "$TEST_TMPDIR/bin/systemctl"
out="$(run_check 2>&1 || true)"
_t_has "$out" 'FAIL stunnel-dot.service not active' "a stopped stunnel is reported"

# Certificate missing: the check that must be run as root to be correct.
rm -f "$CERT_DIR/combined.pem"
printf '#!/usr/bin/env bash\nexit 0\n' >"$TEST_TMPDIR/bin/systemctl"
chmod +x "$TEST_TMPDIR/bin/systemctl"
out="$(run_check 2>&1 || true)"
_t_has "$out" 'FAIL certificate missing' "a missing certificate is reported"

# Nothing listening on the port.
printf '#!/usr/bin/env bash\nexit 0\n' >"$TEST_TMPDIR/bin/ss"
chmod +x "$TEST_TMPDIR/bin/ss"
out="$(run_check 2>&1 || true)"
_t_has "$out" 'FAIL nothing listening' "a silent port is reported"

# kdig present and the DoT query failing. The check is skipped entirely when
# kdig is absent, so this arm only exists on a host that has knot-utils --
# which makes it exactly the arm most likely to rot unnoticed.
printf '#!/usr/bin/env bash\nexit 1\n' >"$TEST_TMPDIR/bin/kdig"
chmod +x "$TEST_TMPDIR/bin/kdig"
printf 'cert\n' >"$CERT_DIR/combined.pem"
printf '#!/usr/bin/env bash\nprintf "LISTEN 0 0 10.8.0.1:%s\\n"\n' "$DOT_PORT" >"$TEST_TMPDIR/bin/ss"
chmod +x "$TEST_TMPDIR/bin/ss"
out="$(run_check 2>&1 || true)"
_t_has "$out" 'FAIL DoT query did not answer' "an unanswered DoT query is reported"

_t_report "test_dot_resolver_install.sh"