# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence. """Coming up, including when signal-cli is not there yet. On 2026-08-30 the machine rebooted, the bot started before the signal-api container was ready, and its very first HTTP call raised. uvicorn reported "Application startup failed", the unit burned its five restarts in 25 seconds, and it sat in `failed` for eight hours while the container had been healthy for almost all of them. Nobody noticed, because a unit nobody is watching is indistinguishable from a quiet group. So the connection to Signal is no longer part of starting up. uvicorn serves first, /healthz answers 503 with the reason, and this module keeps trying in the background until the container answers. A bot that is still connecting is then *visibly* unhealthy rather than absent. The distinction that keeps the systemd start limit meaningful: only errors that another attempt might fix are retried. A container in the wrong mode, or a GROUP_IDS entry naming a group the bot is genuinely not in, are answers -- retrying them forever would replace a visible failure with an invisible one. """ from __future__ import annotations import asyncio import contextlib import logging import signal from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final import httpx from signal_bot.bot import Bot, build_bot, now_local from signal_bot.discovery import GroupDirectory from signal_bot.liveness import Liveness from signal_bot.membership import MembershipError from signal_bot.scheduler import DailyScheduler from signal_bot.transport_rest import RestTransport, TransportError from signal_bot.trust import InviteTrust if TYPE_CHECKING: from signal_bot.config import Config from signal_bot.store import Store logger = logging.getLogger(__name__) FIRST_RETRY_SECONDS: Final = 5.0 # Docker's own restart of signal-api plus signal-cli's account load took # minutes on the reboot this exists for, so the ceiling is what the bot # spends most of that wait at. A minute is short enough that recovery is # still prompt and long enough that a long outage is not a busy loop. MAX_RETRY_SECONDS: Final = 60.0 BACKOFF_FACTOR: Final = 2.0 class NotReadyError(RuntimeError): """Raised when signal-cli answers but cannot yet say what it knows.""" # Another attempt might fix these: the container is still booting, its HTTP # server is up but answering 502, or it is returning a holding page that is # not JSON. ValueError covers the last one -- json.JSONDecodeError is one. RETRYABLE: Final = (httpx.HTTPError, ValueError, NotReadyError) # No further attempt will fix these. They are verdicts, not outages. FATAL: Final = (TransportError, MembershipError) Sleeper = Callable[[float], Awaitable[None]] Stopper = Callable[[], None] @dataclass class Runtime: """The running bot and the background tasks it owns.""" bot: Bot tasks: list[asyncio.Task[None]] = field(default_factory=list) def start(self) -> None: """Spawn the listener and the daily scheduler as background tasks.""" scheduler = DailyScheduler( self.bot.config.report_hour, self.bot.config.report_minute, self.bot.publish_report, now_local, asyncio.sleep, ) self.tasks = [ asyncio.create_task(self.bot.listen()), asyncio.create_task(scheduler.run()), # Third loop: without it, a group added later is only noticed # when somebody happens to speak in it. asyncio.create_task(self.bot.directory.run()), # Fourth loop: without it, a dead connection to Signal is # indistinguishable from a quiet evening, for hours. asyncio.create_task(self.bot.liveness.run()), # Fifth, and one-shot: says hello with a changelog when the # running version is newer than what a group was last told. asyncio.create_task(self.bot.announce_release()), ] async def stop(self) -> None: """Cancel the background tasks. The caller owns the client and store.""" for task in self.tasks: task.cancel() for task in self.tasks: with contextlib.suppress(asyncio.CancelledError): await task async def create_runtime(config: Config, client: httpx.AsyncClient, store: Store) -> Runtime: """Build a Runtime, verifying the container exposes a WebSocket first. Args: config: Resolved runtime configuration. client: HTTP client shared by every component. store: The already-open tally database. Owned by the caller, because this function is retried and must not leak a connection per try. Returns: A Runtime whose loops have not been started yet. Raises: MembershipError: GROUP_IDS names a group the bot is not a member of, established against a listing that was actually read. """ transport = RestTransport(config, client) await transport.verify_mode() directory = GroupDirectory(config, client, store, InviteTrust(config.trusted_inviters)) # Only a listing that really arrived can convict the allowlist. A refresh # that failed leaves the watched set empty, and judging GROUP_IDS on that # would turn "the container is not up yet" into "you are not in your own # group" -- fatal, and wrong. if not await directory.refresh(): msg = f"group listing at {config.groups_url} could not be read" raise NotReadyError(msg) await directory.verify_allowlist() # No probe here: /v1/devices has been seen to hang past 20s, and a # startup that can hang is a startup that can burn the unit's restart # limit. The gate starts healthy and earns its verdict at runtime. liveness = Liveness(config, client) return Runtime(bot=build_bot(config, client, store, transport, directory, liveness)) # Quoted: Config and Store are imported for type checking only, and this # alias is evaluated at import time. Factory = Callable[["Config", httpx.AsyncClient, "Store"], Awaitable[Runtime]] def terminate() -> None: """Ask uvicorn to shut down, from inside a background task. A background task cannot exit the process: SystemExit raised in one only kills that task, is logged as "Task exception was never retrieved", and uvicorn keeps serving a bot that will never connect. SIGTERM is the signal uvicorn already handles, and the unit's start limit then turns the resulting restart loop into a `failed` unit somebody can see. """ signal.raise_signal(signal.SIGTERM) class Starter: """Connects to signal-cli in the background, retrying until it works.""" def __init__( self, config: Config, client: httpx.AsyncClient, store: Store, *, sleeper: Sleeper | None = None, stopper: Stopper | None = None, factory: Factory = create_runtime, ) -> None: """Wire the retry loop to its config and its injectable I/O. Args: config: Resolved runtime configuration. client: HTTP client shared by every component. store: The already-open tally database. sleeper: Async sleep between attempts. Injected by the tests. stopper: How to end the process on a fatal error. Injected by the tests, which must not signal the pytest process. factory: Builds the runtime. Injected by the tests. """ self._config = config self._client = client self._store = store self._sleep = sleeper if sleeper is not None else asyncio.sleep self._stop = stopper if stopper is not None else terminate self._factory = factory self.runtime: Runtime | None = None self._attempts = 0 self._detail = "not attempted yet" self._fatal = False @property def reason(self) -> str: """Why the bot is not connected yet, in one line for /healthz.""" return ( f"connecting to {self._config.api_base_url}, attempt {self._attempts}: {self._detail}" ) async def attempt(self) -> bool: """Try once to build and start the runtime. Returns: True once the bot is connected and its loops are running. """ self._attempts += 1 try: runtime = await self._factory(self._config, self._client, self._store) except FATAL as exc: self._detail = f"{type(exc).__name__}: {exc}" self._fatal = True logger.exception("cannot start, and no retry will help") self._stop() return False except RETRYABLE as exc: self._detail = f"{type(exc).__name__}: {exc}" logger.warning("signal-cli not reachable yet (attempt %d): %s", self._attempts, exc) return False runtime.start() self.runtime = runtime logger.info( "listening as %s after %d attempt(s), watching %d group(s): %r", self._config.phone_number, self._attempts, len(runtime.bot.directory.groups), list(runtime.bot.directory.groups), ) return True async def run(self, iterations: int | None = None) -> None: """Attempt to connect until it works, or until it provably cannot. Args: iterations: Stop after this many failed attempts. None means keep trying forever, which is what production wants. """ completed = 0 delay = FIRST_RETRY_SECONDS while iterations is None or completed < iterations: if await self.attempt() or self._fatal: return await self._sleep(delay) delay = min(delay * BACKOFF_FACTOR, MAX_RETRY_SECONDS) completed += 1