signal-bot/tests/test_schema_tables.py
Krzysztof kuhy Rudnicki 405541be9b
Some checks are pending
CI / check (push) Waiting to run
CI / web (push) Waiting to run
release: 0.2.6 «Stephanus I» — dodano !analityka
One row per command attempt, not two aggregates written only on success.

command_uses holds last-use only and daily_uses counts successes; both are
rate-limiter state and both are written by store.record_use, which the
dispatcher calls only when a handler returns cleanly. A gate refusal and a
crash returned before it, so a command that failed every time looked exactly
like one nobody ran, and nothing knew which command a counted message was.

command_events now takes every attempt with its outcome, its duration and
the item it served; upstream_calls takes every outbound request made while
serving one. The two writes stay separate on purpose -- recording a refusal
as a use is what slid the cooldown window forward every time somebody was
told to wait -- and test_dispatcher_outcomes asserts last_used does not move.

Backfilled from journald joined against message_events, with daily_uses
filling the residual so the two sources cannot double-count. Two things were
wrong on the first attempt and both were found by measuring rather than by
reasoning: the window has to be symmetric, because sent_at_ms comes from the
sender's phone and the journal stamp from this host (194 of 299 lines landed
*before* their own message, by up to 3.2 s); and matching has to be a global
closest-pair assignment, because per-row greedy cannot separate two commands
seconds apart. Together those took the dropped rows from 137 to zero, and
recovered 418 invocations back to 29 August.

Also: !analityka in the group, twelve Polish lines, naming at most three
people and only ones whose display name has been seen; an Analytics tab with
Nivo charts; and continuous latency measurement, which answers the open
question in DOCS-latency.md -- a text send is 235 ms and an image send about
2.2 s, so the upload really is where the time goes.

EPHEMERAL_TABLES was doing two unrelated jobs and is now two tuples: what a
schema bump rebuilds, and what leaving a group clears. daily_uses left the
first (it is a backfill source now) and stayed in the second, and
_drop_reshaped took over the repair a bump used to do for it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T4QTp3h5Q3MvfrwGV8xqPf
2026-08-31 23:19:51 +02:00

186 lines
7.5 KiB
Python

# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Which tables survive what.
EPHEMERAL_TABLES used to answer two different questions with one tuple, and
this file exists because splitting it is exactly the kind of change that looks
right and silently stops clearing something. Every assertion here is against
either sqlite's own catalogue or the code that does the dropping -- never
against a second copy of the list.
"""
from __future__ import annotations
import sqlite3
from typing import TYPE_CHECKING
from signal_bot.schema import (
CLEARED_ON_LEAVE,
PERMANENT_TABLES,
REBUILT_ON_BUMP,
RETAINED_COLUMNS,
SCHEMA_VERSION,
TABLES,
)
from signal_bot.store import Store
if TYPE_CHECKING:
from pathlib import Path
GROUP = "group-a=="
def _declared(store: Store) -> set[str]:
"""Every table sqlite actually holds, as opposed to every one we listed."""
rows = store.events.connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
).fetchall()
return {str(row["name"]) for row in rows}
def _seed(store: Store, group_id: str = GROUP) -> None:
"""One row in every table a group can own."""
store.record_use(group_id, "u1", "cat", 1.0, "2026-08-29")
store.add_vote(group_id, "cat", "u1", 1.0)
store.set_meta(group_id, "last_reported_day", "2026-08-29")
store.suggestions.record_suggestion(group_id, "!pogoda", "na jutro", 1)
store.suggestions.record_unknown(group_id, "!cta", "!cta", 2)
# Both maps are spelled out rather than interpolated, for the reason
# forget_group spells its DELETEs out: a table name in an f-string is a
# SQL-injection shape even when the input is a module constant, and this
# repository takes no lint suppressions. PRAGMA additionally takes no
# parameters at all. test_every_table_can_be_counted keeps COUNTS complete.
COLUMNS = {
"command_uses": "PRAGMA table_info(command_uses)",
"daily_uses": "PRAGMA table_info(daily_uses)",
}
COUNTS = {
"command_uses": "SELECT count(*) AS n FROM command_uses WHERE group_id = ?",
"daily_uses": "SELECT count(*) AS n FROM daily_uses WHERE group_id = ?",
"votes": "SELECT count(*) AS n FROM votes WHERE group_id = ?",
"group_meta": "SELECT count(*) AS n FROM group_meta WHERE group_id = ?",
"message_events": "SELECT count(*) AS n FROM message_events WHERE group_id = ?",
"group_people": "SELECT count(*) AS n FROM group_people WHERE group_id = ?",
"unknown_commands": "SELECT count(*) AS n FROM unknown_commands WHERE group_id = ?",
"suggestions": "SELECT count(*) AS n FROM suggestions WHERE group_id = ?",
"command_events": "SELECT count(*) AS n FROM command_events WHERE group_id = ?",
"upstream_calls": "SELECT count(*) AS n FROM upstream_calls WHERE group_id = ?",
}
def _rows(store: Store, table: str, group_id: str = GROUP) -> int:
"""How many rows one group owns in one table."""
row = store.events.connection.execute(COUNTS[table], (group_id,)).fetchone()
return int(row["n"])
def test_every_table_can_be_counted() -> None:
"""A new table without a count query would skip every check below it."""
assert set(COUNTS) == set(TABLES)
def test_declared_columns_match_the_schema(store: Store) -> None:
"""RETAINED_COLUMNS is a second copy of the DDL, so it must be checked.
_drop_reshaped compares against it to decide whether a retained table is
still writable. A stale entry here would drop a perfectly good table on
every bump -- silently, since dropping is the recovery path -- and take
the use history it was moved out of REBUILT_ON_BUMP to protect.
"""
for table, expected in RETAINED_COLUMNS.items():
found = tuple(str(row["name"]) for row in store.events.connection.execute(COLUMNS[table]))
assert found == expected
def test_a_retained_table_with_the_wrong_shape_is_dropped(tmp_path: Path) -> None:
"""The failure _apply_schema exists to prevent, on a table it no longer rebuilds.
A database written before the group key holds a two-column command_uses.
CREATE TABLE IF NOT EXISTS will not touch it and the bump no longer drops
it, so without _drop_reshaped the bot boots clean and fails on every
INSERT.
"""
path = str(tmp_path / "legacy.sqlite3")
with sqlite3.connect(path) as raw:
raw.execute("CREATE TABLE daily_uses (uuid TEXT PRIMARY KEY, uses INTEGER NOT NULL)")
raw.execute("INSERT INTO daily_uses VALUES ('u1', 7)")
with Store(path) as store:
store.record_use(GROUP, "u1", "cat", 1.0, "2026-08-29")
assert store.uses_today(GROUP, "u1", "cat", "2026-08-29") == 1
def test_the_list_matches_what_sqlite_holds(store: Store) -> None:
"""TABLES is the schema's index, so it may not drift from the schema."""
assert set(TABLES) == _declared(store)
def test_no_table_is_both_kept_and_thrown_away() -> None:
"""A table in two of these tuples is a contradiction, not a configuration."""
assert not set(PERMANENT_TABLES) & set(REBUILT_ON_BUMP)
assert not set(PERMANENT_TABLES) & set(CLEARED_ON_LEAVE)
def test_the_history_is_permanent() -> None:
"""The four tables nothing can recompute, named where it is configured."""
for table in ("message_events", "command_events", "upstream_calls", "group_people"):
assert table in PERMANENT_TABLES
def test_the_use_counters_reset_on_leave_but_survive_a_bump() -> None:
"""The whole point of the split, asserted as one sentence.
daily_uses feeds a rate limiter *and* is a source the analytics history is
backfilled from. Clearing it when the bot leaves a group is correct;
dropping it on a schema bump would throw away use history that predates
command_events.
"""
for table in ("command_uses", "daily_uses"):
assert table in CLEARED_ON_LEAVE
assert table not in REBUILT_ON_BUMP
def test_forget_group_clears_every_cleared_table(store: Store) -> None:
"""The DELETEs in forget_group are spelled out, so something must count them.
Without this, adding a table to CLEARED_ON_LEAVE and forgetting the
matching DELETE leaves a cooldown alive across a leave and a rejoin --
invisible until somebody is told to wait for a command they never ran.
"""
_seed(store)
for table in CLEARED_ON_LEAVE:
assert _rows(store, table) > 0
store.forget_group(GROUP)
for table in CLEARED_ON_LEAVE:
assert _rows(store, table) == 0
def test_forget_group_leaves_the_permanent_tables_alone(store: Store) -> None:
"""A transient group listing must not be able to delete years of history."""
_seed(store)
store.forget_group(GROUP)
assert len(store.suggestions.filed()) == 1
assert len(store.suggestions.unknown()) == 1
def test_a_version_bump_keeps_the_use_history(tmp_path: Path) -> None:
"""A bump drops REBUILT_ON_BUMP and nothing else.
Written against a file rather than :memory: because the bump is only
observable across two openings of the same database.
"""
path = str(tmp_path / "bump.sqlite3")
with Store(path) as store:
_seed(store)
with sqlite3.connect(path) as raw:
raw.execute(f"PRAGMA user_version = {SCHEMA_VERSION - 1}")
with Store(path) as reopened:
# Thrown away, and cheap to lose.
assert _rows(reopened, "votes") == 0
assert _rows(reopened, "group_meta") == 0
# Kept, because nothing recomputes them.
assert _rows(reopened, "command_uses") == 1
assert _rows(reopened, "daily_uses") == 1
assert len(reopened.suggestions.filed()) == 1