signal-bot/scripts/signal_api_watchdog.sh

#!/bin/bash

# ============================================================================
# Restart signal-cli-rest-api when it can no longer reach Signal.
#
# Nothing else notices this failure. On 2026-08-29 the container's own
# healthcheck answered 204, /v1/groups answered 200 out of local state, and
# the bot's WebSocket to it stayed open, while signal-cli reconnected to
# Signal in a loop for three hours and not one message arrived. Docker cannot
# see it, so a timer asks the one question that needs a real round trip.
#
# Runs from signal-api-watchdog.timer as a oneshot, so every counter lives on
# disk: the process has no memory of the previous firing.
# ============================================================================

set -euo pipefail

SCRIPT_NAME="$(basename "$0")"
readonly SCRIPT_NAME

readonly API="${API_BASE_URL:-http://localhost:9922}"
readonly BOT="http://127.0.0.1:${BIND_PORT:-8137}/healthz"
readonly CONTAINER="${SIGNAL_CONTAINER:-signal-api}"
readonly STATE_DIR="${STATE_DIRECTORY:-${XDG_STATE_HOME:-$HOME/.local/state}/signal-api-watchdog}"
# Overridable so the tests can drive the whole ladder without waiting.
readonly PROBE_TIMEOUT="${PROBE_TIMEOUT:-15}"
readonly FAILURES_BEFORE_RESTART="${FAILURES_BEFORE_RESTART:-2}"
readonly MAX_RESTARTS="${MAX_RESTARTS:-3}"
readonly RESTART_WINDOW="${RESTART_WINDOW:-21600}" # 6 hours
readonly SETTLE_SECONDS="${SETTLE_SECONDS:-45}"
# always: every transition pops. escalation: only the give-up does. never: none.
# Default escalation, because the routine restart/recovery pair is exactly the
# noise this knob exists to stop; journald still gets all of it either way.
readonly NOTIFY_MODE="${WATCHDOG_NOTIFY:-escalation}"
readonly NOTIFY_EXPIRE_MS="${WATCHDOG_NOTIFY_EXPIRE_MS:-20000}"

readonly FAILURE_FILE="${STATE_DIR}/consecutive_failures"
readonly RESTART_FILE="${STATE_DIR}/restarts"
readonly NOTIFY_ID_FILE="${STATE_DIR}/notify_id"
readonly ESCALATED_FILE="${STATE_DIR}/escalated_at"

usage() {
	echo "Usage: $SCRIPT_NAME"
	echo "  Probes signal-cli's connection to Signal and restarts ${CONTAINER}"
	echo "  when it has been dead for ${FAILURES_BEFORE_RESTART} consecutive checks."
	echo "  Reads PHONE_NUMBER, API_BASE_URL and BIND_PORT from the environment."
	exit 0
}

die() {
	echo "Error: $1" >&2
	exit 1
}

# The escalation repeats on every firing for as long as the window lasts. The
# journal wants all of it; the desktop wants one box. So the stamp gates the
# POPUP only -- never the log lines, which is why the give-up text still
# appears on stdout for every check that reaches it.
escalation_is_new() {
	local now stamp=0
	now="$(date +%s)"
	if [[ -f "$ESCALATED_FILE" ]]; then
		read -r stamp <"$ESCALATED_FILE" || true
		[[ "$stamp" =~ ^[0-9]+$ ]] || stamp=0
	fi
	if ((now - stamp < RESTART_WINDOW)); then
		return 1
	fi
	printf '%s\n' "$now" >"$ESCALATED_FILE"
}

# One desktop slot, reused. Without --replace-id every firing stacked another
# popup that xfce4-notifyd never expires, so a single outage left a screenful
# of identical boxes to dismiss by hand -- 297 of them in one day.
notify() {
	local urgency="$1" message="$2" previous=0 id
	case "$NOTIFY_MODE" in
	never) return 0 ;;
	escalation)
		if [[ "$urgency" != critical ]]; then
			return 0
		fi
		;;
	*) ;;
	esac
	if [[ "$urgency" == critical ]] && ! escalation_is_new; then
		return 0
	fi
	if [[ -f "$NOTIFY_ID_FILE" ]]; then
		read -r previous <"$NOTIFY_ID_FILE" || true
		[[ "$previous" =~ ^[0-9]+$ ]] || previous=0
	fi
	id="$(notify-send --print-id --replace-id="$previous" --urgency="$urgency" \
		--expire-time="$NOTIFY_EXPIRE_MS" --app-name=signal-bot \
		"signal-bot" "$message" 2>/dev/null)" || return 0
	if [[ "$id" =~ ^[0-9]+$ ]]; then
		printf '%s\n' "$id" >"$NOTIFY_ID_FILE"
	fi
}

# journald and stdout always; the desktop only when notify() agrees. The broken
# path IS Signal, so the bot cannot tell anybody about this itself -- but it
# only needs to say it once.
alarm() {
	logger --priority user.warning --tag signal-api-watchdog -- "$1" || true
	echo "== $1"
	notify "${2:-normal}" "$1"
}

validate_requirements() {
	command -v curl >/dev/null || die "curl not found"
	command -v docker >/dev/null || die "docker not found"
	[[ -n "${PHONE_NUMBER:-}" ]] ||
		die "PHONE_NUMBER is unset -- the unit must load the repo's .env"
	mkdir -p "$STATE_DIR"
}

read_count() {
	local value=0
	[[ -f "$FAILURE_FILE" ]] && read -r value <"$FAILURE_FILE"
	printf '%s' "${value:-0}"
}

# --max-time on every probe: an unbounded curl would outlive the timer
# interval and overlap the next firing.
http_code() {
	curl -s -o /dev/null -m "$2" -w '%{http_code}' "$1" || true
}

# Two tiers. The bot's own /healthz is free and already knows whether it can
# hear; the authenticated probe is what confirms it independently, and what
# still works when the bot process itself is down.
connection_is_alive() {
	if [[ "$(http_code "$BOT" 5)" == "200" ]]; then
		return 0
	fi
	local code
	code="$(http_code "${API}/v1/devices/${PHONE_NUMBER}" "$PROBE_TIMEOUT")"
	if [[ "$code" == "200" ]]; then
		return 0
	fi
	echo "== ${API}/v1/devices answered ${code}"
	return 1
}

# Recovery re-arms the escalation: the next outage is a new event and deserves
# its own box, rather than being silenced by a stamp from the previous one.
clear_alarms() {
	rm -f "$ESCALATED_FILE"
}

record_success() {
	local previous
	previous="$(read_count)"
	printf '0\n' >"$FAILURE_FILE"
	clear_alarms
	if ((previous >= FAILURES_BEFORE_RESTART)); then
		alarm "signal-api is reachable again after ${previous} failed probe(s)."
	fi
}

# Restart history is durable and pruned to the window, so a fault that
# recurs every few hours cannot be papered over by a counter that resets.
recent_restarts() {
	local now cutoff stamp count=0 pruned
	now="$(date +%s)"
	cutoff=$((now - RESTART_WINDOW))
	if [[ ! -f "$RESTART_FILE" ]]; then
		printf '0'
		return 0
	fi
	pruned="${RESTART_FILE}.pruned"
	: >"$pruned"
	while read -r stamp; do
		[[ "$stamp" =~ ^[0-9]+$ ]] || continue
		((stamp > cutoff)) || continue
		printf '%s\n' "$stamp" >>"$pruned"
		count=$((count + 1))
	done <"$RESTART_FILE"
	mv "$pruned" "$RESTART_FILE"
	printf '%s' "$count"
}

give_up() {
	alarm "signal-api has been restarted ${MAX_RESTARTS} time(s) and is still deaf. \
This is probably not transient: the account may be deregistered, or the host \
firewall may be blocking the container. Check 'docker exec ${CONTAINER} head -5 \
/var/log/signal-cli-json-rpc-1/out.log' and 'docker exec ${CONTAINER} getent hosts \
chat.signal.org'. No further restarts until the window passes." critical
}

restart_container() {
	date +%s >>"$RESTART_FILE"
	alarm "signal-api cannot reach Signal; restarting ${CONTAINER}."
	docker restart "$CONTAINER" >/dev/null || die "docker restart ${CONTAINER} failed"
	sleep "$SETTLE_SECONDS"
	if connection_is_alive; then
		printf '0\n' >"$FAILURE_FILE"
		clear_alarms
		alarm "signal-api is back: the connection to Signal is alive again."
		return 0
	fi
	alarm "signal-api restarted but is still deaf; will retry on the next check."
}

main() {
	validate_requirements

	if connection_is_alive; then
		record_success
		exit 0
	fi

	local failures
	failures=$(($(read_count) + 1))
	printf '%s\n' "$failures" >"$FAILURE_FILE"
	logger --priority user.warning --tag signal-api-watchdog -- \
		"signal-api probe failed (${failures}/${FAILURES_BEFORE_RESTART})" || true
	echo "== probe failed (${failures}/${FAILURES_BEFORE_RESTART})"

	if ((failures < FAILURES_BEFORE_RESTART)); then
		exit 0
	fi
	if (($(recent_restarts) >= MAX_RESTARTS)); then
		give_up
		exit 0
	fi
	restart_container
}

while [[ $# -gt 0 ]]; do
	case $1 in
	-h | --help) usage ;;
	*)
		echo "Unknown option: $1" >&2
		exit 1
		;;
	esac
done

main