#!/bin/bash # ============================================================================ # Tests for scripts/signal_api_watchdog.sh. # # Hermetic: curl and docker are both stubbed on PATH, so nothing here talks to # a network or a container. The stub docker can "fix" the stub curl's answers, # which is how the restart-then-recover path is exercised without a restart. # ============================================================================ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" readonly REPO_ROOT readonly WATCHDOG="${REPO_ROOT}/scripts/signal_api_watchdog.sh" WORK="" FAILURES=0 cleanup() { if [[ -n "$WORK" && -d "$WORK" ]]; then rm -rf "$WORK" fi } trap cleanup EXIT fail() { echo " FAIL: $1" >&2 FAILURES=$((FAILURES + 1)) } ok() { echo " ok: $1" } # shellcheck source=scripts/test_watchdog_harness.sh source "${REPO_ROOT}/scripts/test_watchdog_harness.sh" test_healthy_is_free() { echo "== a healthy bot needs no restart and no authenticated probe" setup run_watchdog >/dev/null expect_file "$WORK/state/consecutive_failures" "0" "failure counter stays at zero" if [[ -f "$WORK/docker.log" ]]; then fail "docker was touched while everything was healthy" else ok "docker was never called" fi expect_notifications 0 "a healthy check pops nothing" } test_one_failure_waits() { echo "== one failed probe is not enough to restart anything" setup printf '400' >"$WORK/bot_code" printf '400' >"$WORK/api_code" local out out="$(run_watchdog)" expect_contains "$out" "probe failed (1/2)" "the failure is counted and logged" expect_file "$WORK/state/consecutive_failures" "1" "counter persisted for the next firing" if [[ -f "$WORK/docker.log" ]]; then fail "docker was restarted after a single failure" else ok "docker was not restarted" fi } test_confirmed_failure_restarts_and_recovers() { echo "== two failures restart the container, and recovery is announced" setup printf '400' >"$WORK/bot_code" printf '400' >"$WORK/api_code" printf '1' >"$WORK/fix_on_restart" run_watchdog >/dev/null local out out="$(run_watchdog)" expect_contains "$(cat "$WORK/docker.log")" "restart signal-api" "the container was restarted" expect_contains "$out" "is back" "the recovery is announced" expect_file "$WORK/state/consecutive_failures" "0" "the counter was cleared" expect_notifications 0 "the routine restart/recovery pair stays out of the desktop" } test_restart_that_does_not_help() { echo "== a restart that does not help says so instead of claiming success" setup printf '400' >"$WORK/bot_code" printf '400' >"$WORK/api_code" run_watchdog >/dev/null local out out="$(run_watchdog)" expect_contains "$out" "still deaf" "the failed recovery is reported" } test_gives_up_instead_of_looping() { echo "== it gives up loudly rather than restarting forever" setup printf '400' >"$WORK/bot_code" printf '400' >"$WORK/api_code" local out="" attempt=0 while ((attempt < 6)); do out="$(run_watchdog)" attempt=$((attempt + 1)) done local restarts restarts="$(grep -c "restart signal-api" "$WORK/docker.log")" if [[ "$restarts" == "2" ]]; then ok "restarts stopped at the cap (2)" else fail "expected 2 restarts, got $restarts" fi expect_contains "$out" "still deaf" "the give-up message explains what to check" expect_contains "$out" "out.log" "the give-up message names the log to read" # Three firings reach give_up here; the journal got all three (asserted # above, on the LAST run's stdout) and the desktop got one. expect_notifications 1 "a sustained give-up window pops exactly once" } test_notify_modes() { echo "== WATCHDOG_NOTIFY decides what reaches the desktop" setup printf '400' >"$WORK/bot_code" printf '400' >"$WORK/api_code" WATCHDOG_NOTIFY=never run_watchdog >/dev/null WATCHDOG_NOTIFY=never run_watchdog >/dev/null expect_notifications 0 "never pops nothing at all" setup printf '400' >"$WORK/bot_code" printf '400' >"$WORK/api_code" WATCHDOG_NOTIFY=always run_watchdog >/dev/null WATCHDOG_NOTIFY=always run_watchdog >/dev/null # Two sends -- "restarting" then "still deaf" -- collapsed by --replace-id # into one box on screen. Counting sends is what makes the reuse visible. expect_notifications 2 "always pops the routine pair the default suppresses" expect_contains "$(cat "$WORK/notify.log")" "--replace-id" \ "the popup reuses one desktop slot instead of stacking" expect_file "$WORK/state/notify_id" "4242" "the slot id is persisted for the next firing" } test_requires_a_number() { echo "== it refuses to run without PHONE_NUMBER" setup local out status=0 out="$(PATH="$WORK/bin:$PATH" PHONE_NUMBER="" STATE_DIRECTORY="$WORK/state" \ "$WATCHDOG" 2>&1)" || status=$? if [[ "$status" == "1" ]]; then ok "exited non-zero" else fail "expected exit 1, got $status" fi expect_contains "$out" "PHONE_NUMBER is unset" "it says which value is missing" } test_bot_down_but_account_alive() { echo "== the bot being down is not the container's fault" setup printf '000' >"$WORK/bot_code" local out out="$(run_watchdog)" expect_missing "$out" "probe failed" "the authenticated probe carried the check" if [[ -f "$WORK/docker.log" ]]; then fail "docker was restarted although the account was reachable" else ok "docker was not restarted" fi } main() { WORK="$(mktemp -d)" [[ -d "$WORK" ]] || { echo "Error: failed to create a temporary directory" >&2 exit 1 } echo "============================================================" echo "signal_api_watchdog.sh" echo "============================================================" test_healthy_is_free test_bot_down_but_account_alive test_one_failure_waits test_confirmed_failure_restarts_and_recovers test_restart_that_does_not_help test_gives_up_instead_of_looping test_notify_modes test_requires_a_number echo "============================================================" if ((FAILURES > 0)); then echo "FAILED: $FAILURES check(s)" exit 1 fi echo "all checks passed" } main "$@"