signal-bot/scripts/cold_start.sh

#!/bin/bash

# ============================================================================
# Prove the bot survives being started before Signal exists.
#
# The failure this reproduces: on 2026-08-30 the machine rebooted, the bot
# started before the signal-api container was ready, its first HTTP call
# raised, and systemd's start limit put the unit in `failed` -- where it
# stayed for eight hours while the container had been healthy for most of
# them. Nobody noticed, because nothing was watching the unit.
#
# So: boot the bot with nothing at the other end, then start the fake Signal
# server underneath it and assert the bot recovers on its own. Uses the same
# protocol-faithful fake as e2e.sh. No container, no account, no network.
# ============================================================================

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly SCRIPT_DIR
REPO_ROOT="$(dirname "$SCRIPT_DIR")"
readonly REPO_ROOT
readonly PYTHON="${PYTHON:-$REPO_ROOT/.venv/bin/python}"
readonly FAKE_PORT="${FAKE_PORT:-8097}"
readonly BOT_PORT="${BOT_PORT:-8096}"
readonly FAKE_URL="http://127.0.0.1:${FAKE_PORT}"
readonly HEALTH="http://127.0.0.1:${BOT_PORT}/healthz"
# Four groups, because "rediscovers all four groups" is the acceptance
# condition the real deployment is judged on. Both spellings of each id
# are real base64, exactly as /v1/groups reports them: the canonical
# `group.<b64>` form is the base64 of the internal form.
readonly GROUP_ROWS='[{"id":"group.WTI5c1pDMXZibVV0WjNKdmRYQXRhV1F0TXpJdFlubDBaWE1oSVNFaElTRT0=","internal_id":"Y29sZC1vbmUtZ3JvdXAtaWQtMzItYnl0ZXMhISEhISE=","name":"cold-one","member":true,"joinable":true},{"id":"group.WTI5c1pDMTBkMjh0WjNKdmRYQXRhV1F0TXpJdFlubDBaWE1oSVNFaElTRT0=","internal_id":"Y29sZC10d28tZ3JvdXAtaWQtMzItYnl0ZXMhISEhISE=","name":"cold-two","member":true,"joinable":true},{"id":"group.WTI5c1pDMTBhSEpsWlMxbmNtOTFjQzFwWkMwek1pMWllWFJsY3lFaElTRT0=","internal_id":"Y29sZC10aHJlZS1ncm91cC1pZC0zMi1ieXRlcyEhISE=","name":"cold-three","member":true,"joinable":true},{"id":"group.WTI5c1pDMW1iM1Z5TFdkeWIzVndMV2xrTFRNeUxXSjVkR1Z6SVNFaElTRT0=","internal_id":"Y29sZC1mb3VyLWdyb3VwLWlkLTMyLWJ5dGVzISEhISE=","name":"cold-four","member":true,"joinable":true}]'

FAKE_PID=""
BOT_PID=""
TEMP_DIR=""

cleanup() {
    if [[ -n "$BOT_PID" ]]; then
        kill "$BOT_PID" 2>/dev/null || true
    fi
    if [[ -n "$FAKE_PID" ]]; then
        kill "$FAKE_PID" 2>/dev/null || true
    fi
    wait 2>/dev/null || true
    if [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]]; then
        rm -rf "$TEMP_DIR"
    fi
}
trap cleanup EXIT

die() {
    echo "Error: $1" >&2
    if [[ -n "$TEMP_DIR" && -f "${TEMP_DIR}/bot.log" ]]; then
        echo "--- last 20 lines of the bot log ---" >&2
        tail -20 "${TEMP_DIR}/bot.log" >&2
    fi
    exit 1
}

validate_requirements() {
    if [[ ! -x "$PYTHON" ]]; then
        echo "Error: no interpreter at $PYTHON. Run: uv venv .venv && uv pip install -e '.[dev]'" >&2
        exit 1
    fi
    command -v curl >/dev/null || die "curl is required"
}

health_code() {
    curl -s -o /dev/null -m 5 -w '%{http_code}' "$HEALTH" || true
}

# Poll /healthz until it reports this status, or give up after $2 seconds.
wait_for_status() {
    local want="$1" budget="$2" waited=0 status=""
    while ((waited < budget)); do
        status="$(curl -s -m 5 "$HEALTH" | tr -d ' ' | grep -o '"status":"[a-z]*"' || true)"
        if [[ "$status" == "\"status\":\"${want}\"" ]]; then
            printf '%s' "$waited"
            return 0
        fi
        sleep 1
        waited=$((waited + 1))
    done
    die "/healthz never reported '${want}' (last: ${status:-no answer})"
}

start_the_bot_with_nothing_to_talk_to() {
    echo "== booting the bot on :${BOT_PORT}, pointed at a fake that is not running"
    env -u CAT_API \
        PHONE_NUMBER="+15550000000" \
        GROUP_IDS="" \
        TRUSTED_INVITERS="" \
        API_BASE_URL="$FAKE_URL" \
        DATABASE_PATH="${TEMP_DIR}/cold.sqlite3" \
        BIND_PORT="$BOT_PORT" \
        "$PYTHON" -m signal_bot >"${TEMP_DIR}/bot.log" 2>&1 &
    BOT_PID=$!
}

# Step 2 of the acceptance test: the process must stay up and say why it is
# unhealthy. The old code exited 3 here and systemd gave up 25 seconds later.
prove_it_stays_up_and_says_why() {
    local waited
    waited="$(wait_for_status connecting 30)"
    echo "== /healthz answered 503 'connecting' after ${waited}s"
    kill -0 "$BOT_PID" 2>/dev/null || die "the bot exited instead of waiting"
    [[ "$(health_code)" == "503" ]] || die "'connecting' was not served as 503"
    curl -s "$HEALTH" | grep -q "ConnectError" ||
        die "/healthz went red without naming the connection failure"
}

bring_signal_up_underneath_it() {
    echo "== starting the fake Signal server on :${FAKE_PORT}"
    "$PYTHON" -m uvicorn fake_signal.server:app \
        --host 127.0.0.1 --port "$FAKE_PORT" --log-level warning &
    FAKE_PID=$!
    local attempt=0
    until curl -sf "${FAKE_URL}/v1/about" >/dev/null 2>&1; do
        attempt=$((attempt + 1))
        ((attempt < 100)) || die "the fake never came up"
        sleep 0.1
    done
    curl -sf -X POST "${FAKE_URL}/_control/accounts" \
        -H 'Content-Type: application/json' -d '["+15550000000"]' >/dev/null
    curl -sf -X POST "${FAKE_URL}/_control/groups" \
        -H 'Content-Type: application/json' -d "$GROUP_ROWS" >/dev/null
}

# Step 4: nobody restarts anything. The bot's own retry loop has to notice.
prove_it_recovers_unaided() {
    local waited found
    # The backoff ceiling is 60s, so one full window plus slack.
    waited="$(wait_for_status ok 150)"
    echo "== /healthz recovered on its own after ${waited}s, with no intervention"
    found="$(grep -c 'watching 4 group(s)' "${TEMP_DIR}/bot.log" || true)"
    [[ "$found" != "0" ]] || die "the bot came up without rediscovering all four groups"
    echo "== all four groups rediscovered"
}

main() {
    validate_requirements
    TEMP_DIR="$(mktemp -d)"

    start_the_bot_with_nothing_to_talk_to
    prove_it_stays_up_and_says_why
    bring_signal_up_underneath_it
    prove_it_recovers_unaided

    echo "== cold start passed"
}

main "$@"