mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-10 18:07:29 +01:00
One row per command attempt, not two aggregates written only on success. command_uses holds last-use only and daily_uses counts successes; both are rate-limiter state and both are written by store.record_use, which the dispatcher calls only when a handler returns cleanly. A gate refusal and a crash returned before it, so a command that failed every time looked exactly like one nobody ran, and nothing knew which command a counted message was. command_events now takes every attempt with its outcome, its duration and the item it served; upstream_calls takes every outbound request made while serving one. The two writes stay separate on purpose -- recording a refusal as a use is what slid the cooldown window forward every time somebody was told to wait -- and test_dispatcher_outcomes asserts last_used does not move. Backfilled from journald joined against message_events, with daily_uses filling the residual so the two sources cannot double-count. Two things were wrong on the first attempt and both were found by measuring rather than by reasoning: the window has to be symmetric, because sent_at_ms comes from the sender's phone and the journal stamp from this host (194 of 299 lines landed *before* their own message, by up to 3.2 s); and matching has to be a global closest-pair assignment, because per-row greedy cannot separate two commands seconds apart. Together those took the dropped rows from 137 to zero, and recovered 418 invocations back to 29 August. Also: !analityka in the group, twelve Polish lines, naming at most three people and only ones whose display name has been seen; an Analytics tab with Nivo charts; and continuous latency measurement, which answers the open question in DOCS-latency.md -- a text send is 235 ms and an image send about 2.2 s, so the upload really is where the time goes. EPHEMERAL_TABLES was doing two unrelated jobs and is now two tuples: what a schema bump rebuilds, and what leaving a group clears. daily_uses left the first (it is a backfill source now) and stayed in the second, and _drop_reshaped took over the repair a bump used to do for it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T4QTp3h5Q3MvfrwGV8xqPf
186 lines
7.5 KiB
Python
186 lines
7.5 KiB
Python
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""Which tables survive what.
|
|
|
|
EPHEMERAL_TABLES used to answer two different questions with one tuple, and
|
|
this file exists because splitting it is exactly the kind of change that looks
|
|
right and silently stops clearing something. Every assertion here is against
|
|
either sqlite's own catalogue or the code that does the dropping -- never
|
|
against a second copy of the list.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from typing import TYPE_CHECKING
|
|
|
|
from signal_bot.schema import (
|
|
CLEARED_ON_LEAVE,
|
|
PERMANENT_TABLES,
|
|
REBUILT_ON_BUMP,
|
|
RETAINED_COLUMNS,
|
|
SCHEMA_VERSION,
|
|
TABLES,
|
|
)
|
|
from signal_bot.store import Store
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
GROUP = "group-a=="
|
|
|
|
|
|
def _declared(store: Store) -> set[str]:
|
|
"""Every table sqlite actually holds, as opposed to every one we listed."""
|
|
rows = store.events.connection.execute(
|
|
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
|
|
).fetchall()
|
|
return {str(row["name"]) for row in rows}
|
|
|
|
|
|
def _seed(store: Store, group_id: str = GROUP) -> None:
|
|
"""One row in every table a group can own."""
|
|
store.record_use(group_id, "u1", "cat", 1.0, "2026-08-29")
|
|
store.add_vote(group_id, "cat", "u1", 1.0)
|
|
store.set_meta(group_id, "last_reported_day", "2026-08-29")
|
|
store.suggestions.record_suggestion(group_id, "!pogoda", "na jutro", 1)
|
|
store.suggestions.record_unknown(group_id, "!cta", "!cta", 2)
|
|
|
|
|
|
# Both maps are spelled out rather than interpolated, for the reason
|
|
# forget_group spells its DELETEs out: a table name in an f-string is a
|
|
# SQL-injection shape even when the input is a module constant, and this
|
|
# repository takes no lint suppressions. PRAGMA additionally takes no
|
|
# parameters at all. test_every_table_can_be_counted keeps COUNTS complete.
|
|
COLUMNS = {
|
|
"command_uses": "PRAGMA table_info(command_uses)",
|
|
"daily_uses": "PRAGMA table_info(daily_uses)",
|
|
}
|
|
|
|
COUNTS = {
|
|
"command_uses": "SELECT count(*) AS n FROM command_uses WHERE group_id = ?",
|
|
"daily_uses": "SELECT count(*) AS n FROM daily_uses WHERE group_id = ?",
|
|
"votes": "SELECT count(*) AS n FROM votes WHERE group_id = ?",
|
|
"group_meta": "SELECT count(*) AS n FROM group_meta WHERE group_id = ?",
|
|
"message_events": "SELECT count(*) AS n FROM message_events WHERE group_id = ?",
|
|
"group_people": "SELECT count(*) AS n FROM group_people WHERE group_id = ?",
|
|
"unknown_commands": "SELECT count(*) AS n FROM unknown_commands WHERE group_id = ?",
|
|
"suggestions": "SELECT count(*) AS n FROM suggestions WHERE group_id = ?",
|
|
"command_events": "SELECT count(*) AS n FROM command_events WHERE group_id = ?",
|
|
"upstream_calls": "SELECT count(*) AS n FROM upstream_calls WHERE group_id = ?",
|
|
}
|
|
|
|
|
|
def _rows(store: Store, table: str, group_id: str = GROUP) -> int:
|
|
"""How many rows one group owns in one table."""
|
|
row = store.events.connection.execute(COUNTS[table], (group_id,)).fetchone()
|
|
return int(row["n"])
|
|
|
|
|
|
def test_every_table_can_be_counted() -> None:
|
|
"""A new table without a count query would skip every check below it."""
|
|
assert set(COUNTS) == set(TABLES)
|
|
|
|
|
|
def test_declared_columns_match_the_schema(store: Store) -> None:
|
|
"""RETAINED_COLUMNS is a second copy of the DDL, so it must be checked.
|
|
|
|
_drop_reshaped compares against it to decide whether a retained table is
|
|
still writable. A stale entry here would drop a perfectly good table on
|
|
every bump -- silently, since dropping is the recovery path -- and take
|
|
the use history it was moved out of REBUILT_ON_BUMP to protect.
|
|
"""
|
|
for table, expected in RETAINED_COLUMNS.items():
|
|
found = tuple(str(row["name"]) for row in store.events.connection.execute(COLUMNS[table]))
|
|
assert found == expected
|
|
|
|
|
|
def test_a_retained_table_with_the_wrong_shape_is_dropped(tmp_path: Path) -> None:
|
|
"""The failure _apply_schema exists to prevent, on a table it no longer rebuilds.
|
|
|
|
A database written before the group key holds a two-column command_uses.
|
|
CREATE TABLE IF NOT EXISTS will not touch it and the bump no longer drops
|
|
it, so without _drop_reshaped the bot boots clean and fails on every
|
|
INSERT.
|
|
"""
|
|
path = str(tmp_path / "legacy.sqlite3")
|
|
with sqlite3.connect(path) as raw:
|
|
raw.execute("CREATE TABLE daily_uses (uuid TEXT PRIMARY KEY, uses INTEGER NOT NULL)")
|
|
raw.execute("INSERT INTO daily_uses VALUES ('u1', 7)")
|
|
with Store(path) as store:
|
|
store.record_use(GROUP, "u1", "cat", 1.0, "2026-08-29")
|
|
assert store.uses_today(GROUP, "u1", "cat", "2026-08-29") == 1
|
|
|
|
|
|
def test_the_list_matches_what_sqlite_holds(store: Store) -> None:
|
|
"""TABLES is the schema's index, so it may not drift from the schema."""
|
|
assert set(TABLES) == _declared(store)
|
|
|
|
|
|
def test_no_table_is_both_kept_and_thrown_away() -> None:
|
|
"""A table in two of these tuples is a contradiction, not a configuration."""
|
|
assert not set(PERMANENT_TABLES) & set(REBUILT_ON_BUMP)
|
|
assert not set(PERMANENT_TABLES) & set(CLEARED_ON_LEAVE)
|
|
|
|
|
|
def test_the_history_is_permanent() -> None:
|
|
"""The four tables nothing can recompute, named where it is configured."""
|
|
for table in ("message_events", "command_events", "upstream_calls", "group_people"):
|
|
assert table in PERMANENT_TABLES
|
|
|
|
|
|
def test_the_use_counters_reset_on_leave_but_survive_a_bump() -> None:
|
|
"""The whole point of the split, asserted as one sentence.
|
|
|
|
daily_uses feeds a rate limiter *and* is a source the analytics history is
|
|
backfilled from. Clearing it when the bot leaves a group is correct;
|
|
dropping it on a schema bump would throw away use history that predates
|
|
command_events.
|
|
"""
|
|
for table in ("command_uses", "daily_uses"):
|
|
assert table in CLEARED_ON_LEAVE
|
|
assert table not in REBUILT_ON_BUMP
|
|
|
|
|
|
def test_forget_group_clears_every_cleared_table(store: Store) -> None:
|
|
"""The DELETEs in forget_group are spelled out, so something must count them.
|
|
|
|
Without this, adding a table to CLEARED_ON_LEAVE and forgetting the
|
|
matching DELETE leaves a cooldown alive across a leave and a rejoin --
|
|
invisible until somebody is told to wait for a command they never ran.
|
|
"""
|
|
_seed(store)
|
|
for table in CLEARED_ON_LEAVE:
|
|
assert _rows(store, table) > 0
|
|
store.forget_group(GROUP)
|
|
for table in CLEARED_ON_LEAVE:
|
|
assert _rows(store, table) == 0
|
|
|
|
|
|
def test_forget_group_leaves_the_permanent_tables_alone(store: Store) -> None:
|
|
"""A transient group listing must not be able to delete years of history."""
|
|
_seed(store)
|
|
store.forget_group(GROUP)
|
|
assert len(store.suggestions.filed()) == 1
|
|
assert len(store.suggestions.unknown()) == 1
|
|
|
|
|
|
def test_a_version_bump_keeps_the_use_history(tmp_path: Path) -> None:
|
|
"""A bump drops REBUILT_ON_BUMP and nothing else.
|
|
|
|
Written against a file rather than :memory: because the bump is only
|
|
observable across two openings of the same database.
|
|
"""
|
|
path = str(tmp_path / "bump.sqlite3")
|
|
with Store(path) as store:
|
|
_seed(store)
|
|
with sqlite3.connect(path) as raw:
|
|
raw.execute(f"PRAGMA user_version = {SCHEMA_VERSION - 1}")
|
|
with Store(path) as reopened:
|
|
# Thrown away, and cheap to lose.
|
|
assert _rows(reopened, "votes") == 0
|
|
assert _rows(reopened, "group_meta") == 0
|
|
# Kept, because nothing recomputes them.
|
|
assert _rows(reopened, "command_uses") == 1
|
|
assert _rows(reopened, "daily_uses") == 1
|
|
assert len(reopened.suggestions.filed()) == 1
|