signal-bot/scripts/test_watchdog.sh

#!/bin/bash

# ============================================================================
# Tests for scripts/signal_api_watchdog.sh.
#
# Hermetic: curl and docker are both stubbed on PATH, so nothing here talks to
# a network or a container. The stub docker can "fix" the stub curl's answers,
# which is how the restart-then-recover path is exercised without a restart.
# ============================================================================

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
readonly REPO_ROOT
readonly WATCHDOG="${REPO_ROOT}/scripts/signal_api_watchdog.sh"

WORK=""
FAILURES=0

cleanup() {
	if [[ -n "$WORK" && -d "$WORK" ]]; then
		rm -rf "$WORK"
	fi
}
trap cleanup EXIT

fail() {
	echo "  FAIL: $1" >&2
	FAILURES=$((FAILURES + 1))
}

ok() {
	echo "  ok: $1"
}

# Rebuild a pristine sandbox: stub binaries, stub answers, empty state.
setup() {
	rm -rf "$WORK"
	mkdir -p "$WORK/bin" "$WORK/state"
	printf '200' >"$WORK/bot_code"
	printf '200' >"$WORK/api_code"
	printf '0' >"$WORK/fix_on_restart"

	cat >"$WORK/bin/curl" <<EOF
#!/bin/bash
url="\${*: -1}"
case "\$url" in
*/healthz) cat "$WORK/bot_code" ;;
*) cat "$WORK/api_code" ;;
esac
EOF

	cat >"$WORK/bin/docker" <<EOF
#!/bin/bash
echo "\$@" >> "$WORK/docker.log"
if [[ "\$(cat "$WORK/fix_on_restart")" == "1" ]]; then
	printf '200' > "$WORK/bot_code"
	printf '200' > "$WORK/api_code"
fi
EOF
	chmod +x "$WORK/bin/curl" "$WORK/bin/docker"
}

run_watchdog() {
	PATH="$WORK/bin:$PATH" \
		PHONE_NUMBER="+15550000000" \
		STATE_DIRECTORY="$WORK/state" \
		SETTLE_SECONDS=0 \
		FAILURES_BEFORE_RESTART=2 \
		MAX_RESTARTS=2 \
		RESTART_WINDOW=3600 \
		"$WATCHDOG" 2>&1
}

expect_contains() {
	if [[ "$1" == *"$2"* ]]; then
		ok "$3"
	else
		fail "$3 -- got: $1"
	fi
}

expect_missing() {
	if [[ "$1" != *"$2"* ]]; then
		ok "$3"
	else
		fail "$3 -- unexpectedly found $2"
	fi
}

expect_file() {
	local actual
	actual="$(cat "$1" 2>/dev/null || echo MISSING)"
	if [[ "$actual" == "$2" ]]; then
		ok "$3"
	else
		fail "$3 -- $1 is '$actual', expected '$2'"
	fi
}

test_healthy_is_free() {
	echo "== a healthy bot needs no restart and no authenticated probe"
	setup
	run_watchdog >/dev/null
	expect_file "$WORK/state/consecutive_failures" "0" "failure counter stays at zero"
	if [[ -f "$WORK/docker.log" ]]; then
		fail "docker was touched while everything was healthy"
	else
		ok "docker was never called"
	fi
}

test_one_failure_waits() {
	echo "== one failed probe is not enough to restart anything"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	local out
	out="$(run_watchdog)"
	expect_contains "$out" "probe failed (1/2)" "the failure is counted and logged"
	expect_file "$WORK/state/consecutive_failures" "1" "counter persisted for the next firing"
	if [[ -f "$WORK/docker.log" ]]; then
		fail "docker was restarted after a single failure"
	else
		ok "docker was not restarted"
	fi
}

test_confirmed_failure_restarts_and_recovers() {
	echo "== two failures restart the container, and recovery is announced"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	printf '1' >"$WORK/fix_on_restart"
	run_watchdog >/dev/null
	local out
	out="$(run_watchdog)"
	expect_contains "$(cat "$WORK/docker.log")" "restart signal-api" "the container was restarted"
	expect_contains "$out" "is back" "the recovery is announced"
	expect_file "$WORK/state/consecutive_failures" "0" "the counter was cleared"
}

test_restart_that_does_not_help() {
	echo "== a restart that does not help says so instead of claiming success"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	run_watchdog >/dev/null
	local out
	out="$(run_watchdog)"
	expect_contains "$out" "still deaf" "the failed recovery is reported"
}

test_gives_up_instead_of_looping() {
	echo "== it gives up loudly rather than restarting forever"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	local out="" attempt=0
	while ((attempt < 6)); do
		out="$(run_watchdog)"
		attempt=$((attempt + 1))
	done
	local restarts
	restarts="$(grep -c "restart signal-api" "$WORK/docker.log")"
	if [[ "$restarts" == "2" ]]; then
		ok "restarts stopped at the cap (2)"
	else
		fail "expected 2 restarts, got $restarts"
	fi
	expect_contains "$out" "still deaf" "the give-up message explains what to check"
	expect_contains "$out" "out.log" "the give-up message names the log to read"
}

test_requires_a_number() {
	echo "== it refuses to run without PHONE_NUMBER"
	setup
	local out status=0
	out="$(PATH="$WORK/bin:$PATH" PHONE_NUMBER="" STATE_DIRECTORY="$WORK/state" \
		"$WATCHDOG" 2>&1)" || status=$?
	if [[ "$status" == "1" ]]; then
		ok "exited non-zero"
	else
		fail "expected exit 1, got $status"
	fi
	expect_contains "$out" "PHONE_NUMBER is unset" "it says which value is missing"
}

test_bot_down_but_account_alive() {
	echo "== the bot being down is not the container's fault"
	setup
	printf '000' >"$WORK/bot_code"
	local out
	out="$(run_watchdog)"
	expect_missing "$out" "probe failed" "the authenticated probe carried the check"
	if [[ -f "$WORK/docker.log" ]]; then
		fail "docker was restarted although the account was reachable"
	else
		ok "docker was not restarted"
	fi
}

main() {
	WORK="$(mktemp -d)"
	[[ -d "$WORK" ]] || {
		echo "Error: failed to create a temporary directory" >&2
		exit 1
	}

	echo "============================================================"
	echo "signal_api_watchdog.sh"
	echo "============================================================"
	test_healthy_is_free
	test_bot_down_but_account_alive
	test_one_failure_waits
	test_confirmed_failure_restarts_and_recovers
	test_restart_that_does_not_help
	test_gives_up_instead_of_looping
	test_requires_a_number

	echo "============================================================"
	if ((FAILURES > 0)); then
		echo "FAILED: $FAILURES check(s)"
		exit 1
	fi
	echo "all checks passed"
}

main "$@"