signal-bot/signal_bot/analytics/backfill.py

# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Recovering the command history that predates the table holding it.

Two sources, and they overlap, which is the whole difficulty. Summing them
would roughly double every historical count and nothing would fail.

* **journald** knows *which* command ran and *when*, to the second, but has
  no group and no uuid -- ``dispatcher.py`` logs a display name because the
  line was written to be read by a human.
* **daily_uses** knows the group, the uuid, the command and the day, and
  counts them correctly, but has no times of day at all.

So the join runs over ``message_events``, which already holds the group, the
uuid, the real hour and the display name of every counted command -- and
lacks only the command's *name*, which is exactly what journald has. Each
matched row is a whole truthful record. ``daily_uses`` then fills the
**residual**: for each (group, user, command, day), whatever its count says
happened beyond what the join could account for. Nothing is counted twice.

Insert-only, always. There is no delete-then-rebuild pass, because journald
has no retention configuration and will rotate: a second run after the log
has aged out would delete rows it could no longer replace. Re-runnability
comes from the natural key and ``ON CONFLICT DO NOTHING``.

One inaccuracy, stated rather than hidden. ``dispatcher.py`` logs "running
command X" *before* the handler runs, so a journald row proves the command
was entered, not that it finished. A handler that then crashed, or a
``daily_limit`` or ``vote_gated`` refusal -- neither of which logged
anything -- is imported as ``ok`` because nothing in the record says
otherwise. Measured against the live database on 2026-08-31 the gap is about
4%: 293 reconstructed invocations against 388 successes in ``daily_uses``,
overshooting by roughly fifteen rows across all commands. Every such row
carries ``source = 'journald'``, so the UI can and does say where it came
from; only the ``live`` rows have outcomes that were actually observed.
"""

from __future__ import annotations

import datetime as dt
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final

from signal_bot.analytics.backfill_join import Tally, from_journal
from signal_bot.analytics.uses import (
    OUTCOME_OK,
    SOURCE_DAILY,
    SOURCE_LIVE,
    CommandEvent,
)
from signal_bot.stats.backfill import UNKNOWN_HOUR
from signal_bot.stats.periods import MS_PER_SECOND, WARSAW, weekday_of

if TYPE_CHECKING:
    from collections.abc import Iterable

    from signal_bot.analytics.journal import JournalLine
    from signal_bot.store import Store

# Recorded per group once a pass has run, in the same place and for the same
# reason stats/backfill.py records its own marker.
BACKFILL_KEY: Final = "command_backfill"


@dataclass(frozen=True, slots=True)
class Report:
    """What one pass did, and what it refused to guess at."""

    journal_rows: int = 0
    daily_rows: int = 0
    unmatched: int = 0
    ambiguous: int = 0
    groups: int = 0

    def lines(self) -> tuple[str, ...]:
        """The summary a human reads. Never names anybody."""
        return (
            f"journald rows:      {self.journal_rows}",
            f"daily_uses rows:    {self.daily_rows}",
            f"dropped, no match:  {self.unmatched}",
            f"dropped, ambiguous: {self.ambiguous}",
            f"groups marked:      {self.groups}",
        )


def midnight_ms(day: str) -> int:
    """The first millisecond of a Warsaw calendar day.

    Built from the date rather than by subtracting 86_400_000 from anything:
    Warsaw days are not all 24 hours, and the arithmetic spelling lands an
    hour out twice a year.
    """
    start = dt.datetime.combine(dt.date.fromisoformat(day), dt.time.min, tzinfo=WARSAW)
    return int(start.timestamp() * MS_PER_SECOND)


def live_floor(store: Store) -> int:
    """The earliest live row, or "no live rows yet".

    Every backfilled row must be strictly older than this. Without the
    cutoff, a second pass run after live recording had started would import
    journald lines describing invocations that already exist as ``live``
    rows -- with a different source, so the natural key would not stop them.
    """
    row = store.commands.connection.execute(
        "SELECT MIN(invoked_at_ms) AS floor FROM command_events WHERE source = ?",
        (SOURCE_LIVE,),
    ).fetchone()
    floor = row["floor"]
    # No live rows yet means nothing is off limits.
    return int(floor) if floor is not None else 1 << 62


def _residual(store: Store, tally: Tally) -> list[CommandEvent]:
    """Fill in whatever daily_uses counted that the join could not explain.

    ``uses`` is the truth about how many times something ran that day. The
    join accounts for some of them exactly; this adds the remainder at day
    granularity, with no hour and no durations, so the total is right even
    where the detail is missing.
    """
    events: list[CommandEvent] = []
    rows = store.commands.connection.execute(
        "SELECT group_id, user_uuid, command, day, uses FROM daily_uses ORDER BY day, command"
    ).fetchall()
    for row in rows:
        key = (str(row["group_id"]), str(row["user_uuid"]), str(row["command"]), str(row["day"]))
        remaining = int(row["uses"]) - tally.matched[key]
        day = str(row["day"])
        base = midnight_ms(day)
        events.extend(
            CommandEvent(
                group_id=key[0],
                user_uuid=key[1],
                command=key[2],
                outcome=OUTCOME_OK,
                # Deterministic, so a second pass produces the identical key
                # and inserts nothing. The same trick, for the same reason,
                # as seed_from_counter's sent_at_ms=index + 1.
                invoked_at_ms=base + index,
                day=day,
                local_hour=UNKNOWN_HOUR,
                weekday=weekday_of(day),
                source=SOURCE_DAILY,
            )
            for index in range(max(0, remaining))
        )
    return events


def plan(store: Store, lines: Iterable[JournalLine]) -> tuple[Report, list[CommandEvent]]:
    """Work out what would be imported, writing nothing.

    Separate from ``apply`` so that ``--dry-run`` reports the real numbers
    rather than an estimate of them. Rolling back would not have worked:
    ``CommandEventStore.record`` commits per row, exactly as ``EventStore``
    does, so by the time a caller could roll back there is nothing left to
    roll back.
    """
    floor = live_floor(store)
    tally = from_journal(store, lines, floor)
    residual = _residual(store, tally)
    events = [*tally.events, *residual]
    return (
        Report(
            journal_rows=len(tally.events),
            daily_rows=len(residual),
            unmatched=tally.unmatched,
            ambiguous=tally.ambiguous,
            groups=len({event.group_id for event in events}),
        ),
        events,
    )


def apply(store: Store, events: Iterable[CommandEvent]) -> None:
    """Write a planned import. Insert-only; a repeat run inserts nothing."""
    groups = set()
    for event in events:
        store.commands.record(event)
        groups.add(event.group_id)
    for group_id in sorted(groups):
        store.set_meta(group_id, BACKFILL_KEY, "done")