mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-21 11:02:21 +01:00
One row per command attempt, not two aggregates written only on success. command_uses holds last-use only and daily_uses counts successes; both are rate-limiter state and both are written by store.record_use, which the dispatcher calls only when a handler returns cleanly. A gate refusal and a crash returned before it, so a command that failed every time looked exactly like one nobody ran, and nothing knew which command a counted message was. command_events now takes every attempt with its outcome, its duration and the item it served; upstream_calls takes every outbound request made while serving one. The two writes stay separate on purpose -- recording a refusal as a use is what slid the cooldown window forward every time somebody was told to wait -- and test_dispatcher_outcomes asserts last_used does not move. Backfilled from journald joined against message_events, with daily_uses filling the residual so the two sources cannot double-count. Two things were wrong on the first attempt and both were found by measuring rather than by reasoning: the window has to be symmetric, because sent_at_ms comes from the sender's phone and the journal stamp from this host (194 of 299 lines landed *before* their own message, by up to 3.2 s); and matching has to be a global closest-pair assignment, because per-row greedy cannot separate two commands seconds apart. Together those took the dropped rows from 137 to zero, and recovered 418 invocations back to 29 August. Also: !analityka in the group, twelve Polish lines, naming at most three people and only ones whose display name has been seen; an Analytics tab with Nivo charts; and continuous latency measurement, which answers the open question in DOCS-latency.md -- a text send is 235 ms and an image send about 2.2 s, so the upload really is where the time goes. EPHEMERAL_TABLES was doing two unrelated jobs and is now two tuples: what a schema bump rebuilds, and what leaving a group clears. daily_uses left the first (it is a backfill source now) and stayed in the second, and _drop_reshaped took over the repair a bump used to do for it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T4QTp3h5Q3MvfrwGV8xqPf
181 lines
7.3 KiB
Python
181 lines
7.3 KiB
Python
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""The database shape, kept apart from the code that queries it.
|
|
|
|
Split out of ``store.py`` so both stay under the 250-line cap, and because
|
|
"what the tables are" and "how the bot reads them" are two things.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Final
|
|
|
|
# Bumped whenever the shape below changes. See Store._apply_schema for why
|
|
# a version number is the only safe way to notice.
|
|
#
|
|
# Deliberately NOT bumped when a whole new table is added: CREATE TABLE IF
|
|
# NOT EXISTS is additive and harmless, whereas a bump drops every table and
|
|
# would throw away the live tallies to gain nothing.
|
|
SCHEMA_VERSION: Final = 2
|
|
|
|
# Every table is keyed by group first. The bot watches several groups, and a
|
|
# tally, a cooldown or a vote that leaked between them would mean a test
|
|
# message throttling the production group -- so the scope is in the primary
|
|
# key rather than in the callers' discipline.
|
|
SCHEMA: Final = """
|
|
CREATE TABLE IF NOT EXISTS command_uses (
|
|
group_id TEXT NOT NULL,
|
|
user_uuid TEXT NOT NULL,
|
|
command TEXT NOT NULL,
|
|
used_at REAL NOT NULL,
|
|
PRIMARY KEY (group_id, user_uuid, command)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS daily_uses (
|
|
group_id TEXT NOT NULL,
|
|
user_uuid TEXT NOT NULL,
|
|
command TEXT NOT NULL,
|
|
day TEXT NOT NULL,
|
|
uses INTEGER NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (group_id, user_uuid, command, day)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS votes (
|
|
group_id TEXT NOT NULL,
|
|
command TEXT NOT NULL,
|
|
voter_uuid TEXT NOT NULL,
|
|
voted_at REAL NOT NULL,
|
|
PRIMARY KEY (group_id, command, voter_uuid)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS group_meta (
|
|
group_id TEXT NOT NULL,
|
|
key TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
PRIMARY KEY (group_id, key)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS message_events (
|
|
group_id TEXT NOT NULL,
|
|
uuid TEXT NOT NULL,
|
|
display_name TEXT NOT NULL,
|
|
sent_at_ms INTEGER NOT NULL,
|
|
day TEXT NOT NULL,
|
|
local_hour INTEGER NOT NULL,
|
|
media_kind TEXT NOT NULL DEFAULT '',
|
|
media_count INTEGER NOT NULL DEFAULT 0,
|
|
char_len INTEGER NOT NULL DEFAULT 0,
|
|
is_command INTEGER NOT NULL DEFAULT 0,
|
|
is_reply INTEGER NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (group_id, uuid, sent_at_ms)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_events_group_day
|
|
ON message_events (group_id, day);
|
|
CREATE INDEX IF NOT EXISTS idx_events_group_uuid_day
|
|
ON message_events (group_id, uuid, day);
|
|
CREATE TABLE IF NOT EXISTS group_people (
|
|
group_id TEXT NOT NULL,
|
|
uuid TEXT NOT NULL,
|
|
display_name TEXT NOT NULL,
|
|
number TEXT NOT NULL DEFAULT '',
|
|
first_seen TEXT NOT NULL,
|
|
last_seen TEXT NOT NULL,
|
|
PRIMARY KEY (group_id, uuid)
|
|
);
|
|
-- Keyed on the envelope's own sent timestamp, never on the wall clock.
|
|
-- signal-cli replays undelivered frames when the bot reconnects; a clock
|
|
-- reading is fresh on every replay, so a clock-keyed table collects a
|
|
-- duplicate row per reconnect instead of ignoring one. message_events
|
|
-- learned this already -- see its ON CONFLICT DO NOTHING.
|
|
CREATE TABLE IF NOT EXISTS unknown_commands (
|
|
group_id TEXT NOT NULL,
|
|
word TEXT NOT NULL,
|
|
body TEXT NOT NULL,
|
|
sent_at_ms INTEGER NOT NULL,
|
|
PRIMARY KEY (group_id, word, sent_at_ms)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_unknown_group_word
|
|
ON unknown_commands (group_id, word);
|
|
CREATE TABLE IF NOT EXISTS suggestions (
|
|
group_id TEXT NOT NULL,
|
|
command TEXT NOT NULL,
|
|
body TEXT NOT NULL,
|
|
sent_at_ms INTEGER NOT NULL,
|
|
PRIMARY KEY (group_id, command, sent_at_ms)
|
|
);
|
|
"""
|
|
|
|
# These were one tuple, EPHEMERAL_TABLES, until command analytics needed the
|
|
# two jobs it was doing to come apart. "Dropped when the schema changes" and
|
|
# "cleared when the bot leaves a group" happened to name the same four tables,
|
|
# so one constant served both -- and then daily_uses became a source the
|
|
# analytics history is backfilled from, which must survive a schema bump but
|
|
# must still reset a rejoined group's daily limits. Two questions, two tuples.
|
|
|
|
# Dropped and recreated when SCHEMA_VERSION rises. Only tables whose contents
|
|
# cost nothing to lose: a vote in progress, a per-group marker.
|
|
#
|
|
# The price of taking command_uses/daily_uses out of here is explicit: if
|
|
# their *shape* ever changes, CREATE TABLE IF NOT EXISTS is a no-op against
|
|
# the old one and every INSERT starts failing. Reshaping either of them now
|
|
# needs a written migration rather than a version bump. That is the trade
|
|
# that keeps the use history.
|
|
REBUILT_ON_BUMP: Final = (
|
|
"votes",
|
|
"group_meta",
|
|
)
|
|
# Rows deleted for one group when the bot is removed from it, so that being
|
|
# added back starts with a clean cooldown rather than resurrecting whatever
|
|
# was true whenever it was last in there.
|
|
CLEARED_ON_LEAVE: Final = (
|
|
"command_uses",
|
|
"daily_uses",
|
|
"votes",
|
|
"group_meta",
|
|
)
|
|
# Never dropped and never cleared, by any path. These hold the whole
|
|
# statistical history, which cannot be recomputed from anything -- so a
|
|
# transient group listing that omits a member (the exact failure
|
|
# discovery._permitted guards against) must not be able to delete years of
|
|
# it, and neither must a future schema bump that only meant to reshape a
|
|
# cooldown table.
|
|
PERMANENT_TABLES: Final = (
|
|
"message_events",
|
|
"group_people",
|
|
# What the group wished existed. Neither can be recomputed from anything
|
|
# -- an idea somebody typed once is gone the moment it is dropped -- and
|
|
# forget_group() clears its own set on the transient group-listing gap
|
|
# discovery._permitted guards against, which must not cost a year of
|
|
# collected suggestions.
|
|
"unknown_commands",
|
|
"suggestions",
|
|
# Every command attempt ever made, and every upstream call made serving
|
|
# one. In neither tuple above: a schema bump must not throw away the
|
|
# answer to "which command is dying", and leaving a group must not
|
|
# either -- message_events already survives both for the same reason.
|
|
"command_events",
|
|
"upstream_calls",
|
|
)
|
|
# Every table the schema declares. command_uses and daily_uses are in
|
|
# CLEARED_ON_LEAVE but not PERMANENT_TABLES, so the union is spelled out
|
|
# rather than concatenated -- test_schema_tables checks it against sqlite's
|
|
# own catalogue, which is what keeps this honest.
|
|
TABLES: Final = (
|
|
"command_uses",
|
|
"daily_uses",
|
|
*REBUILT_ON_BUMP,
|
|
*PERMANENT_TABLES,
|
|
)
|
|
|
|
# The two tables that carry history and are NOT rebuilt on a version bump.
|
|
# Because they survive, a bump can no longer fix them if their shape is
|
|
# wrong, and CREATE TABLE IF NOT EXISTS will not either -- which is the
|
|
# failure _apply_schema exists to prevent: a bot that boots clean and then
|
|
# fails on every INSERT. So their columns are declared here and checked, and
|
|
# a mismatch drops the table after all. Losing a cooldown is recoverable;
|
|
# a bot that answers nothing is not.
|
|
#
|
|
# This is the "written migration" the comment on REBUILT_ON_BUMP promises.
|
|
# A deliberate reshape means editing the tuple below in the same commit, and
|
|
# accepting that the old rows go. test_declared_columns_match_the_schema
|
|
# asserts these against what sqlite actually creates, so they cannot drift.
|
|
RETAINED_COLUMNS: Final = {
|
|
"command_uses": ("group_id", "user_uuid", "command", "used_at"),
|
|
"daily_uses": ("group_id", "user_uuid", "command", "day", "uses"),
|
|
}
|