signal-bot/signal_bot/startup.py

# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Coming up, including when signal-cli is not there yet.

On 2026-08-30 the machine rebooted, the bot started before the signal-api
container was ready, and its very first HTTP call raised. uvicorn reported
"Application startup failed", the unit burned its five restarts in 25
seconds, and it sat in `failed` for eight hours while the container had been
healthy for almost all of them. Nobody noticed, because a unit nobody is
watching is indistinguishable from a quiet group.

So the connection to Signal is no longer part of starting up. uvicorn serves
first, /healthz answers 503 with the reason, and this module keeps trying in
the background until the container answers. A bot that is still connecting is
then *visibly* unhealthy rather than absent.

The distinction that keeps the systemd start limit meaningful: only errors
that another attempt might fix are retried. A container in the wrong mode, or
a GROUP_IDS entry naming a group the bot is genuinely not in, are answers --
retrying them forever would replace a visible failure with an invisible one.
"""

from __future__ import annotations

import asyncio
import contextlib
import logging
import signal
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Final

import httpx

from signal_bot.bot import Bot, build_bot, now_local
from signal_bot.discovery import GroupDirectory
from signal_bot.liveness import Liveness
from signal_bot.membership import MembershipError
from signal_bot.scheduler import DailyScheduler
from signal_bot.transport_rest import RestTransport, TransportError
from signal_bot.trust import InviteTrust

if TYPE_CHECKING:
    from signal_bot.config import Config
    from signal_bot.store import Store

logger = logging.getLogger(__name__)

FIRST_RETRY_SECONDS: Final = 5.0
# Docker's own restart of signal-api plus signal-cli's account load took
# minutes on the reboot this exists for, so the ceiling is what the bot
# spends most of that wait at. A minute is short enough that recovery is
# still prompt and long enough that a long outage is not a busy loop.
MAX_RETRY_SECONDS: Final = 60.0
BACKOFF_FACTOR: Final = 2.0


class NotReadyError(RuntimeError):
    """Raised when signal-cli answers but cannot yet say what it knows."""


# Another attempt might fix these: the container is still booting, its HTTP
# server is up but answering 502, or it is returning a holding page that is
# not JSON. ValueError covers the last one -- json.JSONDecodeError is one.
RETRYABLE: Final = (httpx.HTTPError, ValueError, NotReadyError)
# No further attempt will fix these. They are verdicts, not outages.
FATAL: Final = (TransportError, MembershipError)

Sleeper = Callable[[float], Awaitable[None]]
Stopper = Callable[[], None]


@dataclass
class Runtime:
    """The running bot and the background tasks it owns."""

    bot: Bot
    tasks: list[asyncio.Task[None]] = field(default_factory=list)

    def start(self) -> None:
        """Spawn the listener and the daily scheduler as background tasks."""
        scheduler = DailyScheduler(
            self.bot.config.report_hour,
            self.bot.config.report_minute,
            self.bot.publish_report,
            now_local,
            asyncio.sleep,
        )
        self.tasks = [
            asyncio.create_task(self.bot.listen()),
            asyncio.create_task(scheduler.run()),
            # Third loop: without it, a group added later is only noticed
            # when somebody happens to speak in it.
            asyncio.create_task(self.bot.directory.run()),
            # Fourth loop: without it, a dead connection to Signal is
            # indistinguishable from a quiet evening, for hours.
            asyncio.create_task(self.bot.liveness.run()),
            # Fifth, and one-shot: says hello with a changelog when the
            # running version is newer than what a group was last told.
            asyncio.create_task(self.bot.announce_release()),
        ]

    async def stop(self) -> None:
        """Cancel the background tasks. The caller owns the client and store."""
        for task in self.tasks:
            task.cancel()
        for task in self.tasks:
            with contextlib.suppress(asyncio.CancelledError):
                await task


async def create_runtime(config: Config, client: httpx.AsyncClient, store: Store) -> Runtime:
    """Build a Runtime, verifying the container exposes a WebSocket first.

    Args:
        config: Resolved runtime configuration.
        client: HTTP client shared by every component.
        store: The already-open tally database. Owned by the caller, because
            this function is retried and must not leak a connection per try.

    Returns:
        A Runtime whose loops have not been started yet.

    Raises:
        MembershipError: GROUP_IDS names a group the bot is not a member of,
            established against a listing that was actually read.
    """
    transport = RestTransport(config, client)
    await transport.verify_mode()
    directory = GroupDirectory(config, client, store, InviteTrust(config.trusted_inviters))
    # Only a listing that really arrived can convict the allowlist. A refresh
    # that failed leaves the watched set empty, and judging GROUP_IDS on that
    # would turn "the container is not up yet" into "you are not in your own
    # group" -- fatal, and wrong.
    if not await directory.refresh():
        msg = f"group listing at {config.groups_url} could not be read"
        raise NotReadyError(msg)
    await directory.verify_allowlist()
    # No probe here: /v1/devices has been seen to hang past 20s, and a
    # startup that can hang is a startup that can burn the unit's restart
    # limit. The gate starts healthy and earns its verdict at runtime.
    liveness = Liveness(config, client)
    return Runtime(bot=build_bot(config, client, store, transport, directory, liveness))


# Quoted: Config and Store are imported for type checking only, and this
# alias is evaluated at import time.
Factory = Callable[["Config", httpx.AsyncClient, "Store"], Awaitable[Runtime]]


def terminate() -> None:
    """Ask uvicorn to shut down, from inside a background task.

    A background task cannot exit the process: SystemExit raised in one only
    kills that task, is logged as "Task exception was never retrieved", and
    uvicorn keeps serving a bot that will never connect. SIGTERM is the
    signal uvicorn already handles, and the unit's start limit then turns the
    resulting restart loop into a `failed` unit somebody can see.
    """
    signal.raise_signal(signal.SIGTERM)


class Starter:
    """Connects to signal-cli in the background, retrying until it works."""

    def __init__(
        self,
        config: Config,
        client: httpx.AsyncClient,
        store: Store,
        *,
        sleeper: Sleeper | None = None,
        stopper: Stopper | None = None,
        factory: Factory = create_runtime,
    ) -> None:
        """Wire the retry loop to its config and its injectable I/O.

        Args:
            config: Resolved runtime configuration.
            client: HTTP client shared by every component.
            store: The already-open tally database.
            sleeper: Async sleep between attempts. Injected by the tests.
            stopper: How to end the process on a fatal error. Injected by
                the tests, which must not signal the pytest process.
            factory: Builds the runtime. Injected by the tests.
        """
        self._config = config
        self._client = client
        self._store = store
        self._sleep = sleeper if sleeper is not None else asyncio.sleep
        self._stop = stopper if stopper is not None else terminate
        self._factory = factory
        self.runtime: Runtime | None = None
        self._attempts = 0
        self._detail = "not attempted yet"
        self._fatal = False

    @property
    def reason(self) -> str:
        """Why the bot is not connected yet, in one line for /healthz."""
        return (
            f"connecting to {self._config.api_base_url}, attempt {self._attempts}: {self._detail}"
        )

    async def attempt(self) -> bool:
        """Try once to build and start the runtime.

        Returns:
            True once the bot is connected and its loops are running.
        """
        self._attempts += 1
        try:
            runtime = await self._factory(self._config, self._client, self._store)
        except FATAL as exc:
            self._detail = f"{type(exc).__name__}: {exc}"
            self._fatal = True
            logger.exception("cannot start, and no retry will help")
            self._stop()
            return False
        except RETRYABLE as exc:
            self._detail = f"{type(exc).__name__}: {exc}"
            logger.warning("signal-cli not reachable yet (attempt %d): %s", self._attempts, exc)
            return False
        runtime.start()
        self.runtime = runtime
        logger.info(
            "listening as %s after %d attempt(s), watching %d group(s): %r",
            self._config.phone_number,
            self._attempts,
            len(runtime.bot.directory.groups),
            list(runtime.bot.directory.groups),
        )
        return True

    async def run(self, iterations: int | None = None) -> None:
        """Attempt to connect until it works, or until it provably cannot.

        Args:
            iterations: Stop after this many failed attempts. None means
                keep trying forever, which is what production wants.
        """
        completed = 0
        delay = FIRST_RETRY_SECONDS
        while iterations is None or completed < iterations:
            if await self.attempt() or self._fatal:
                return
            await self._sleep(delay)
            delay = min(delay * BACKOFF_FACTOR, MAX_RETRY_SECONDS)
            completed += 1