signal-bot/signal_bot/startup_retry.py

# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Retrying the connection to signal-cli until it works, in the background.

Split out of ``startup.py`` so both stay under the 250-line cap. ``startup``
owns *what a running bot is* (:class:`~signal_bot.startup.Runtime` and how it
is built); this module owns *getting there* -- the retry loop, its backoff, and
the one distinction that keeps the systemd start limit meaningful: only errors
that another attempt might fix are retried. A container in the wrong mode, or a
GROUP_IDS entry naming a group the bot is genuinely not in, are answers.
Retrying them forever would replace a visible failure with an invisible one.
"""

from __future__ import annotations

import asyncio
import logging
import signal
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final

import httpx

from signal_bot.membership import MembershipError
from signal_bot.startup import NotReadyError, Runtime, create_runtime
from signal_bot.transport_rest import TransportError

if TYPE_CHECKING:
    from signal_bot.config import Config
    from signal_bot.store import Store

logger = logging.getLogger(__name__)

FIRST_RETRY_SECONDS: Final = 5.0
# Docker's own restart of signal-api plus signal-cli's account load took
# minutes on the reboot this exists for, so the ceiling is what the bot
# spends most of that wait at. A minute is short enough that recovery is
# still prompt and long enough that a long outage is not a busy loop.
MAX_RETRY_SECONDS: Final = 60.0
BACKOFF_FACTOR: Final = 2.0

# Another attempt might fix these: the container is still booting, its HTTP
# server is up but answering 502, or it is returning a holding page that is
# not JSON. ValueError covers the last one -- json.JSONDecodeError is one.
#
# Note what else ValueError covers: registry.DuplicateTriggerError. A command
# alias claimed twice is a programming error that no retry can fix, but it is
# raised while build_bot runs inside create_runtime, so it lands here and the
# bot retries forever reporting "signal-cli not reachable yet" -- the wrong
# diagnosis entirely. tests/test_stats_commands.py builds the whole registry
# so that collision is caught at test time instead.
RETRYABLE: Final = (httpx.HTTPError, ValueError, NotReadyError)
# No further attempt will fix these. They are verdicts, not outages.
FATAL: Final = (TransportError, MembershipError)

Sleeper = Callable[[float], Awaitable[None]]
Stopper = Callable[[], None]

# Quoted: Config and Store are imported for type checking only, and this
# alias is evaluated at import time.
Factory = Callable[["Config", httpx.AsyncClient, "Store"], Awaitable[Runtime]]


def terminate() -> None:
    """Ask uvicorn to shut down, from inside a background task.

    A background task cannot exit the process: SystemExit raised in one only
    kills that task, is logged as "Task exception was never retrieved", and
    uvicorn keeps serving a bot that will never connect. SIGTERM is the
    signal uvicorn already handles, and the unit's start limit then turns the
    resulting restart loop into a `failed` unit somebody can see.
    """
    signal.raise_signal(signal.SIGTERM)


class Starter:
    """Connects to signal-cli in the background, retrying until it works."""

    def __init__(
        self,
        config: Config,
        client: httpx.AsyncClient,
        store: Store,
        *,
        sleeper: Sleeper | None = None,
        stopper: Stopper | None = None,
        factory: Factory = create_runtime,
    ) -> None:
        """Wire the retry loop to its config and its injectable I/O.

        Args:
            config: Resolved runtime configuration.
            client: HTTP client shared by every component.
            store: The already-open event database.
            sleeper: Async sleep between attempts. Injected by the tests.
            stopper: How to end the process on a fatal error. Injected by
                the tests, which must not signal the pytest process.
            factory: Builds the runtime. Injected by the tests.
        """
        self._config = config
        self._client = client
        self._store = store
        self._sleep = sleeper if sleeper is not None else asyncio.sleep
        self._stop = stopper if stopper is not None else terminate
        self._factory = factory
        self.runtime: Runtime | None = None
        self._attempts = 0
        self._detail = "not attempted yet"
        self._fatal = False

    @property
    def reason(self) -> str:
        """Why the bot is not connected yet, in one line for /healthz."""
        return (
            f"connecting to {self._config.api_base_url}, attempt {self._attempts}: {self._detail}"
        )

    async def attempt(self) -> bool:
        """Try once to build and start the runtime.

        Returns:
            True once the bot is connected and its loops are running.
        """
        self._attempts += 1
        try:
            runtime = await self._factory(self._config, self._client, self._store)
        except FATAL as exc:
            self._detail = f"{type(exc).__name__}: {exc}"
            self._fatal = True
            logger.exception("cannot start, and no retry will help")
            self._stop()
            return False
        except RETRYABLE as exc:
            self._detail = f"{type(exc).__name__}: {exc}"
            logger.warning("signal-cli not reachable yet (attempt %d): %s", self._attempts, exc)
            return False
        runtime.start()
        self.runtime = runtime
        logger.info(
            "listening as %s after %d attempt(s), watching %d group(s): %r",
            self._config.phone_number,
            self._attempts,
            len(runtime.bot.directory.groups),
            list(runtime.bot.directory.groups),
        )
        return True

    async def run(self, iterations: int | None = None) -> None:
        """Attempt to connect until it works, or until it provably cannot.

        Args:
            iterations: Stop after this many failed attempts. None means
                keep trying forever, which is what production wants.
        """
        completed = 0
        delay = FIRST_RETRY_SECONDS
        while iterations is None or completed < iterations:
            if await self.attempt() or self._fatal:
                return
            await self._sleep(delay)
            delay = min(delay * BACKOFF_FACTOR, MAX_RETRY_SECONDS)
            completed += 1