mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-10 19:17:02 +01:00
Krzysztof kuhy Rudnicki
405541be9b
One row per command attempt, not two aggregates written only on success. command_uses holds last-use only and daily_uses counts successes; both are rate-limiter state and both are written by store.record_use, which the dispatcher calls only when a handler returns cleanly. A gate refusal and a crash returned before it, so a command that failed every time looked exactly like one nobody ran, and nothing knew which command a counted message was. command_events now takes every attempt with its outcome, its duration and the item it served; upstream_calls takes every outbound request made while serving one. The two writes stay separate on purpose -- recording a refusal as a use is what slid the cooldown window forward every time somebody was told to wait -- and test_dispatcher_outcomes asserts last_used does not move. Backfilled from journald joined against message_events, with daily_uses filling the residual so the two sources cannot double-count. Two things were wrong on the first attempt and both were found by measuring rather than by reasoning: the window has to be symmetric, because sent_at_ms comes from the sender's phone and the journal stamp from this host (194 of 299 lines landed *before* their own message, by up to 3.2 s); and matching has to be a global closest-pair assignment, because per-row greedy cannot separate two commands seconds apart. Together those took the dropped rows from 137 to zero, and recovered 418 invocations back to 29 August. Also: !analityka in the group, twelve Polish lines, naming at most three people and only ones whose display name has been seen; an Analytics tab with Nivo charts; and continuous latency measurement, which answers the open question in DOCS-latency.md -- a text send is 235 ms and an image send about 2.2 s, so the upload really is where the time goes. EPHEMERAL_TABLES was doing two unrelated jobs and is now two tuples: what a schema bump rebuilds, and what leaving a group clears. daily_uses left the first (it is a backfill source now) and stayed in the second, and _drop_reshaped took over the repair a bump used to do for it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T4QTp3h5Q3MvfrwGV8xqPf
107 lines
3.5 KiB
Python
107 lines
3.5 KiB
Python
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""Family F: which items get served, and how often the same one comes back.
|
|
|
|
Reads ``item_id``, which holds public identifiers only -- a Wikidata QID, a
|
|
Commons page id, a Wolne Lektury slug, a content-addressed filename. Never a
|
|
title, a caption or a name. That constraint is what makes this family
|
|
possible at all: "the pope pool feels repetitive" is an impression, and
|
|
"106 draws produced 61 distinct popes" is a number.
|
|
|
|
The repeat rate is the headline. For a pool of size N drawn uniformly, the
|
|
expected number of distinct items after k draws is N(1 - (1 - 1/N)^k), so a
|
|
distinct count well below that means the draw is not uniform -- which is
|
|
exactly the bug that put one identical photograph into a live group chat
|
|
when iNaturalist silently ignored ``order_by=random``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Final
|
|
|
|
if TYPE_CHECKING:
|
|
import sqlite3
|
|
|
|
from signal_bot.analytics.scope import Scope
|
|
|
|
ITEMS_SQL: Final = """
|
|
SELECT command,
|
|
COUNT(*) AS draws,
|
|
COUNT(DISTINCT item_id) AS distinct_items
|
|
FROM command_events
|
|
WHERE (:group IS NULL OR group_id = :group)
|
|
AND item_id IS NOT NULL
|
|
AND outcome = 'ok'
|
|
GROUP BY command
|
|
ORDER BY draws DESC, command
|
|
"""
|
|
|
|
TOP_ITEMS_SQL: Final = """
|
|
SELECT command, item_id, COUNT(*) AS draws
|
|
FROM command_events
|
|
WHERE (:group IS NULL OR group_id = :group)
|
|
AND item_id IS NOT NULL
|
|
AND outcome = 'ok'
|
|
GROUP BY command, item_id
|
|
HAVING draws > 1
|
|
ORDER BY draws DESC, command, item_id
|
|
"""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Pool:
|
|
"""One command's draws, and how varied they turned out to be."""
|
|
|
|
command: str
|
|
draws: int
|
|
distinct: int
|
|
|
|
@property
|
|
def repeat_rate(self) -> float:
|
|
"""Share of draws that produced something already seen.
|
|
|
|
Zero means every draw was new. Approaching one means the pool is
|
|
exhausted -- or was never as large as it looked.
|
|
"""
|
|
if self.draws <= 0:
|
|
return 0.0
|
|
return 1.0 - (self.distinct / self.draws)
|
|
|
|
|
|
def pools(conn: sqlite3.Connection, scope: Scope) -> list[Pool]:
|
|
"""Every command that names what it served, most drawn first."""
|
|
return [
|
|
Pool(
|
|
command=str(row["command"]),
|
|
draws=int(row["draws"]),
|
|
distinct=int(row["distinct_items"]),
|
|
)
|
|
for row in conn.execute(ITEMS_SQL, scope.params())
|
|
]
|
|
|
|
|
|
def repeats(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, str, int]]:
|
|
"""(command, item, times) for every item served more than once.
|
|
|
|
The identifiers are safe to show -- that is the whole reason the column
|
|
may exist -- but they are only meaningful next to the command, so they
|
|
are never returned on their own.
|
|
"""
|
|
return [
|
|
(str(row["command"]), str(row["item_id"]), int(row["draws"]))
|
|
for row in conn.execute(TOP_ITEMS_SQL, scope.params())
|
|
]
|
|
|
|
|
|
def expected_distinct(pool_size: int, draws: int) -> float:
|
|
"""How many distinct items a *uniform* draw would have produced.
|
|
|
|
The comparison that turns a repeat rate into a verdict. A pool of 266
|
|
traps drawn 19 times should yield about 18.4 distinct ones; yielding 9
|
|
would mean the randomness is not what it claims, which has happened in
|
|
this repository before and reached a live group chat.
|
|
"""
|
|
if pool_size <= 0 or draws <= 0:
|
|
return 0.0
|
|
return pool_size * (1.0 - ((pool_size - 1) / pool_size) ** draws)
|