signal-bot/scripts/signal_api_watchdog.sh

#!/bin/bash

# ============================================================================
# Restart signal-cli-rest-api when it can no longer reach Signal.
#
# Nothing else notices this failure. On 2026-08-29 the container's own
# healthcheck answered 204, /v1/groups answered 200 out of local state, and
# the bot's WebSocket to it stayed open, while signal-cli reconnected to
# Signal in a loop for three hours and not one message arrived. Docker cannot
# see it, so a timer asks the one question that needs a real round trip.
#
# Runs from signal-api-watchdog.timer as a oneshot, so every counter lives on
# disk: the process has no memory of the previous firing.
# ============================================================================

set -euo pipefail

SCRIPT_NAME="$(basename "$0")"
readonly SCRIPT_NAME

readonly API="${API_BASE_URL:-http://localhost:9922}"
readonly BOT="http://127.0.0.1:${BIND_PORT:-8137}/healthz"
readonly CONTAINER="${SIGNAL_CONTAINER:-signal-api}"
readonly STATE_DIR="${STATE_DIRECTORY:-${XDG_STATE_HOME:-$HOME/.local/state}/signal-api-watchdog}"
# Overridable so the tests can drive the whole ladder without waiting.
readonly PROBE_TIMEOUT="${PROBE_TIMEOUT:-15}"
readonly FAILURES_BEFORE_RESTART="${FAILURES_BEFORE_RESTART:-2}"
readonly MAX_RESTARTS="${MAX_RESTARTS:-3}"
readonly RESTART_WINDOW="${RESTART_WINDOW:-21600}" # 6 hours
readonly SETTLE_SECONDS="${SETTLE_SECONDS:-45}"

readonly FAILURE_FILE="${STATE_DIR}/consecutive_failures"
readonly RESTART_FILE="${STATE_DIR}/restarts"

usage() {
	echo "Usage: $SCRIPT_NAME"
	echo "  Probes signal-cli's connection to Signal and restarts ${CONTAINER}"
	echo "  when it has been dead for ${FAILURES_BEFORE_RESTART} consecutive checks."
	echo "  Reads PHONE_NUMBER, API_BASE_URL and BIND_PORT from the environment."
	exit 0
}

die() {
	echo "Error: $1" >&2
	exit 1
}

# journald and the desktop, always both: the broken path IS Signal, so the
# bot cannot tell anybody about this itself.
alarm() {
	logger --priority user.warning --tag signal-api-watchdog -- "$1" || true
	echo "== $1"
	notify-send --urgency=critical --app-name=signal-bot "signal-bot" "$1" 2>/dev/null || true
}

validate_requirements() {
	command -v curl >/dev/null || die "curl not found"
	command -v docker >/dev/null || die "docker not found"
	[[ -n "${PHONE_NUMBER:-}" ]] ||
		die "PHONE_NUMBER is unset -- the unit must load the repo's .env"
	mkdir -p "$STATE_DIR"
}

read_count() {
	local value=0
	[[ -f "$FAILURE_FILE" ]] && read -r value <"$FAILURE_FILE"
	printf '%s' "${value:-0}"
}

# --max-time on every probe: an unbounded curl would outlive the timer
# interval and overlap the next firing.
http_code() {
	curl -s -o /dev/null -m "$2" -w '%{http_code}' "$1" || true
}

# Two tiers. The bot's own /healthz is free and already knows whether it can
# hear; the authenticated probe is what confirms it independently, and what
# still works when the bot process itself is down.
connection_is_alive() {
	if [[ "$(http_code "$BOT" 5)" == "200" ]]; then
		return 0
	fi
	local code
	code="$(http_code "${API}/v1/devices/${PHONE_NUMBER}" "$PROBE_TIMEOUT")"
	if [[ "$code" == "200" ]]; then
		return 0
	fi
	echo "== ${API}/v1/devices answered ${code}"
	return 1
}

record_success() {
	local previous
	previous="$(read_count)"
	printf '0\n' >"$FAILURE_FILE"
	if ((previous >= FAILURES_BEFORE_RESTART)); then
		alarm "signal-api is reachable again after ${previous} failed probe(s)."
	fi
}

# Restart history is durable and pruned to the window, so a fault that
# recurs every few hours cannot be papered over by a counter that resets.
recent_restarts() {
	local now cutoff stamp count=0 pruned
	now="$(date +%s)"
	cutoff=$((now - RESTART_WINDOW))
	if [[ ! -f "$RESTART_FILE" ]]; then
		printf '0'
		return 0
	fi
	pruned="${RESTART_FILE}.pruned"
	: >"$pruned"
	while read -r stamp; do
		[[ "$stamp" =~ ^[0-9]+$ ]] || continue
		((stamp > cutoff)) || continue
		printf '%s\n' "$stamp" >>"$pruned"
		count=$((count + 1))
	done <"$RESTART_FILE"
	mv "$pruned" "$RESTART_FILE"
	printf '%s' "$count"
}

give_up() {
	alarm "signal-api has been restarted ${MAX_RESTARTS} time(s) and is still deaf. \
This is probably not transient: the account may be deregistered, or the host \
firewall may be blocking the container. Check 'docker exec ${CONTAINER} head -5 \
/var/log/signal-cli-json-rpc-1/out.log' and 'docker exec ${CONTAINER} getent hosts \
chat.signal.org'. No further restarts until the window passes."
}

restart_container() {
	date +%s >>"$RESTART_FILE"
	alarm "signal-api cannot reach Signal; restarting ${CONTAINER}."
	docker restart "$CONTAINER" >/dev/null || die "docker restart ${CONTAINER} failed"
	sleep "$SETTLE_SECONDS"
	if connection_is_alive; then
		printf '0\n' >"$FAILURE_FILE"
		alarm "signal-api is back: the connection to Signal is alive again."
		return 0
	fi
	alarm "signal-api restarted but is still deaf; will retry on the next check."
}

main() {
	validate_requirements

	if connection_is_alive; then
		record_success
		exit 0
	fi

	local failures
	failures=$(($(read_count) + 1))
	printf '%s\n' "$failures" >"$FAILURE_FILE"
	logger --priority user.warning --tag signal-api-watchdog -- \
		"signal-api probe failed (${failures}/${FAILURES_BEFORE_RESTART})" || true
	echo "== probe failed (${failures}/${FAILURES_BEFORE_RESTART})"

	if ((failures < FAILURES_BEFORE_RESTART)); then
		exit 0
	fi
	if (($(recent_restarts) >= MAX_RESTARTS)); then
		give_up
		exit 0
	fi
	restart_container
}

while [[ $# -gt 0 ]]; do
	case $1 in
	-h | --help) usage ;;
	*)
		echo "Unknown option: $1" >&2
		exit 1
		;;
	esac
done

main