signal-bot/scripts/test_watchdog.sh

#!/bin/bash

# ============================================================================
# Tests for scripts/signal_api_watchdog.sh.
#
# Hermetic: curl and docker are both stubbed on PATH, so nothing here talks to
# a network or a container. The stub docker can "fix" the stub curl's answers,
# which is how the restart-then-recover path is exercised without a restart.
# ============================================================================

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
readonly REPO_ROOT
readonly WATCHDOG="${REPO_ROOT}/scripts/signal_api_watchdog.sh"

WORK=""
FAILURES=0

cleanup() {
	if [[ -n "$WORK" && -d "$WORK" ]]; then
		rm -rf "$WORK"
	fi
}
trap cleanup EXIT

fail() {
	echo "  FAIL: $1" >&2
	FAILURES=$((FAILURES + 1))
}

ok() {
	echo "  ok: $1"
}

# shellcheck source=scripts/test_watchdog_harness.sh
source "${REPO_ROOT}/scripts/test_watchdog_harness.sh"

test_healthy_is_free() {
	echo "== a healthy bot needs no restart and no authenticated probe"
	setup
	run_watchdog >/dev/null
	expect_file "$WORK/state/consecutive_failures" "0" "failure counter stays at zero"
	if [[ -f "$WORK/docker.log" ]]; then
		fail "docker was touched while everything was healthy"
	else
		ok "docker was never called"
	fi
	expect_notifications 0 "a healthy check pops nothing"
}

test_one_failure_waits() {
	echo "== one failed probe is not enough to restart anything"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	local out
	out="$(run_watchdog)"
	expect_contains "$out" "probe failed (1/2)" "the failure is counted and logged"
	expect_file "$WORK/state/consecutive_failures" "1" "counter persisted for the next firing"
	if [[ -f "$WORK/docker.log" ]]; then
		fail "docker was restarted after a single failure"
	else
		ok "docker was not restarted"
	fi
}

test_confirmed_failure_restarts_and_recovers() {
	echo "== two failures restart the container, and recovery is announced"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	printf '1' >"$WORK/fix_on_restart"
	run_watchdog >/dev/null
	local out
	out="$(run_watchdog)"
	expect_contains "$(cat "$WORK/docker.log")" "restart signal-api" "the container was restarted"
	expect_contains "$out" "is back" "the recovery is announced"
	expect_file "$WORK/state/consecutive_failures" "0" "the counter was cleared"
	expect_notifications 0 "the routine restart/recovery pair stays out of the desktop"
}

test_restart_that_does_not_help() {
	echo "== a restart that does not help says so instead of claiming success"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	run_watchdog >/dev/null
	local out
	out="$(run_watchdog)"
	expect_contains "$out" "still deaf" "the failed recovery is reported"
}

test_gives_up_instead_of_looping() {
	echo "== it gives up loudly rather than restarting forever"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	local out="" attempt=0
	while ((attempt < 6)); do
		out="$(run_watchdog)"
		attempt=$((attempt + 1))
	done
	local restarts
	restarts="$(grep -c "restart signal-api" "$WORK/docker.log")"
	if [[ "$restarts" == "2" ]]; then
		ok "restarts stopped at the cap (2)"
	else
		fail "expected 2 restarts, got $restarts"
	fi
	expect_contains "$out" "still deaf" "the give-up message explains what to check"
	expect_contains "$out" "out.log" "the give-up message names the log to read"
	# Three firings reach give_up here; the journal got all three (asserted
	# above, on the LAST run's stdout) and the desktop got one.
	expect_notifications 1 "a sustained give-up window pops exactly once"
}

test_notify_modes() {
	echo "== WATCHDOG_NOTIFY decides what reaches the desktop"
	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	WATCHDOG_NOTIFY=never run_watchdog >/dev/null
	WATCHDOG_NOTIFY=never run_watchdog >/dev/null
	expect_notifications 0 "never pops nothing at all"

	setup
	printf '400' >"$WORK/bot_code"
	printf '400' >"$WORK/api_code"
	WATCHDOG_NOTIFY=always run_watchdog >/dev/null
	WATCHDOG_NOTIFY=always run_watchdog >/dev/null
	# Two sends -- "restarting" then "still deaf" -- collapsed by --replace-id
	# into one box on screen. Counting sends is what makes the reuse visible.
	expect_notifications 2 "always pops the routine pair the default suppresses"
	expect_contains "$(cat "$WORK/notify.log")" "--replace-id" \
		"the popup reuses one desktop slot instead of stacking"
	expect_file "$WORK/state/notify_id" "4242" "the slot id is persisted for the next firing"
}

test_requires_a_number() {
	echo "== it refuses to run without PHONE_NUMBER"
	setup
	local out status=0
	out="$(PATH="$WORK/bin:$PATH" PHONE_NUMBER="" STATE_DIRECTORY="$WORK/state" \
		"$WATCHDOG" 2>&1)" || status=$?
	if [[ "$status" == "1" ]]; then
		ok "exited non-zero"
	else
		fail "expected exit 1, got $status"
	fi
	expect_contains "$out" "PHONE_NUMBER is unset" "it says which value is missing"
}

test_bot_down_but_account_alive() {
	echo "== the bot being down is not the container's fault"
	setup
	printf '000' >"$WORK/bot_code"
	local out
	out="$(run_watchdog)"
	expect_missing "$out" "probe failed" "the authenticated probe carried the check"
	if [[ -f "$WORK/docker.log" ]]; then
		fail "docker was restarted although the account was reachable"
	else
		ok "docker was not restarted"
	fi
}

main() {
	WORK="$(mktemp -d)"
	[[ -d "$WORK" ]] || {
		echo "Error: failed to create a temporary directory" >&2
		exit 1
	}

	echo "============================================================"
	echo "signal_api_watchdog.sh"
	echo "============================================================"
	test_healthy_is_free
	test_bot_down_but_account_alive
	test_one_failure_waits
	test_confirmed_failure_restarts_and_recovers
	test_restart_that_does_not_help
	test_gives_up_instead_of_looping
	test_notify_modes
	test_requires_a_number

	echo "============================================================"
	if ((FAILURES > 0)); then
		echo "FAILED: $FAILURES check(s)"
		exit 1
	fi
	echo "all checks passed"
}

main "$@"