mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-10 19:07:19 +01:00
Krzysztof kuhy Rudnicki
405541be9b
One row per command attempt, not two aggregates written only on success. command_uses holds last-use only and daily_uses counts successes; both are rate-limiter state and both are written by store.record_use, which the dispatcher calls only when a handler returns cleanly. A gate refusal and a crash returned before it, so a command that failed every time looked exactly like one nobody ran, and nothing knew which command a counted message was. command_events now takes every attempt with its outcome, its duration and the item it served; upstream_calls takes every outbound request made while serving one. The two writes stay separate on purpose -- recording a refusal as a use is what slid the cooldown window forward every time somebody was told to wait -- and test_dispatcher_outcomes asserts last_used does not move. Backfilled from journald joined against message_events, with daily_uses filling the residual so the two sources cannot double-count. Two things were wrong on the first attempt and both were found by measuring rather than by reasoning: the window has to be symmetric, because sent_at_ms comes from the sender's phone and the journal stamp from this host (194 of 299 lines landed *before* their own message, by up to 3.2 s); and matching has to be a global closest-pair assignment, because per-row greedy cannot separate two commands seconds apart. Together those took the dropped rows from 137 to zero, and recovered 418 invocations back to 29 August. Also: !analityka in the group, twelve Polish lines, naming at most three people and only ones whose display name has been seen; an Analytics tab with Nivo charts; and continuous latency measurement, which answers the open question in DOCS-latency.md -- a text send is 235 ms and an image send about 2.2 s, so the upload really is where the time goes. EPHEMERAL_TABLES was doing two unrelated jobs and is now two tuples: what a schema bump rebuilds, and what leaving a group clears. daily_uses left the first (it is a backfill source now) and stayed in the second, and _drop_reshaped took over the repair a bump used to do for it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T4QTp3h5Q3MvfrwGV8xqPf
90 lines
3.2 KiB
Python
Executable File
90 lines
3.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""Import the command history that predates the table holding it.
|
|
|
|
python tool/backfill_analytics.py --db signal_bot.sqlite3 --dry-run
|
|
python tool/backfill_analytics.py --db signal_bot.sqlite3
|
|
|
|
Two sources, joined so that neither double-counts the other -- see
|
|
``signal_bot.analytics.backfill`` for why, which is the part worth reading
|
|
before trusting the numbers this prints.
|
|
|
|
The logic lives in ``signal_bot/``, where the 100% coverage gate can see it.
|
|
This file is the acquisition and the argparse, and lives in ``tool/``, which
|
|
is outside ``--cov`` and covered instead by ``tests/test_tool_imports.py``.
|
|
|
|
``--db`` is required and has no default, for the reason ManageDeps gives:
|
|
this writes to a database that holds a real group's history, and a default
|
|
is how the wrong one gets written to.
|
|
|
|
Run it against a copy first. It is insert-only and re-runnable, but "I ran
|
|
it on the live file to see what it would do" is not a thing to want back.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from signal_bot.analytics import backfill, journal
|
|
from signal_bot.external import run
|
|
from signal_bot.store import open_store
|
|
|
|
UNIT = "signal-bot.service"
|
|
|
|
|
|
def read_journal(path: str | None) -> list[str]:
|
|
"""The bot's log, from a file or from journald itself.
|
|
|
|
Shells out through ``signal_bot.external``, the one module in this
|
|
repository allowed to start a process. ``tests/test_external.py`` keeps
|
|
that list closed by grepping every file for the import itself, so this
|
|
one routes through the sanctioned seam rather than joining the list --
|
|
and note that naming the import in prose is enough to trip that grep.
|
|
"""
|
|
if path is not None:
|
|
return Path(path).read_text(encoding="utf-8").splitlines()
|
|
result = run(
|
|
["journalctl", "--user", "-u", UNIT, "--no-pager", "-o", "short-iso"],
|
|
timeout=120.0,
|
|
)
|
|
if not result.ok:
|
|
sys.stderr.write(f"could not read the journal: {result.stderr.strip()}\n")
|
|
return []
|
|
return result.stdout.splitlines()
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
"""Parse arguments, run one pass, and print what it did."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--db", required=True, help="the database to write to")
|
|
parser.add_argument("--log", help="a captured journal, instead of reading journald")
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="report what would be imported, and write nothing",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
lines = read_journal(args.log)
|
|
sys.stdout.write(f"journal lines read: {len(lines)}\n")
|
|
records = list(journal.parse(lines))
|
|
sys.stdout.write(f"command lines found: {len(records)}\n")
|
|
|
|
with open_store(args.db) as store:
|
|
report, events = backfill.plan(store, records)
|
|
if args.dry_run:
|
|
sys.stdout.write("-- dry run, nothing written --\n")
|
|
else:
|
|
backfill.apply(store, events)
|
|
for line in report.lines():
|
|
sys.stdout.write(f"{line}\n")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|