signal-bot/tests/test_analytics_usage.py
Krzysztof kuhy Rudnicki 405541be9b
Some checks are pending
CI / check (push) Waiting to run
CI / web (push) Waiting to run
release: 0.2.6 «Stephanus I» — dodano !analityka
One row per command attempt, not two aggregates written only on success.

command_uses holds last-use only and daily_uses counts successes; both are
rate-limiter state and both are written by store.record_use, which the
dispatcher calls only when a handler returns cleanly. A gate refusal and a
crash returned before it, so a command that failed every time looked exactly
like one nobody ran, and nothing knew which command a counted message was.

command_events now takes every attempt with its outcome, its duration and
the item it served; upstream_calls takes every outbound request made while
serving one. The two writes stay separate on purpose -- recording a refusal
as a use is what slid the cooldown window forward every time somebody was
told to wait -- and test_dispatcher_outcomes asserts last_used does not move.

Backfilled from journald joined against message_events, with daily_uses
filling the residual so the two sources cannot double-count. Two things were
wrong on the first attempt and both were found by measuring rather than by
reasoning: the window has to be symmetric, because sent_at_ms comes from the
sender's phone and the journal stamp from this host (194 of 299 lines landed
*before* their own message, by up to 3.2 s); and matching has to be a global
closest-pair assignment, because per-row greedy cannot separate two commands
seconds apart. Together those took the dropped rows from 137 to zero, and
recovered 418 invocations back to 29 August.

Also: !analityka in the group, twelve Polish lines, naming at most three
people and only ones whose display name has been seen; an Analytics tab with
Nivo charts; and continuous latency measurement, which answers the open
question in DOCS-latency.md -- a text send is 235 ms and an image send about
2.2 s, so the upload really is where the time goes.

EPHEMERAL_TABLES was doing two unrelated jobs and is now two tuples: what a
schema bump rebuilds, and what leaving a group clears. daily_uses left the
first (it is a backfill source now) and stayed in the second, and
_drop_reshaped took over the repair a bump used to do for it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T4QTp3h5Q3MvfrwGV8xqPf
2026-08-31 23:19:51 +02:00

160 lines
5.8 KiB
Python

# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family A, and the scope every family shares."""
from __future__ import annotations
from typing import TYPE_CHECKING
from signal_bot.analytics import usage
from signal_bot.analytics.scope import Scope, window
from signal_bot.analytics.uses import OUTCOME_COOLDOWN, OUTCOME_ERROR, OUTCOME_UNKNOWN
from tests.analytics_seed import ALA, BASIA, GROUP, OTHER, Shape, backfilled, ran, reconstructed
if TYPE_CHECKING:
import sqlite3
from signal_bot.store import Store
DAY = "2026-08-29"
NEXT = "2026-08-30"
def conn_of(store: Store) -> sqlite3.Connection:
"""The connection the query layer reads through."""
return store.commands.connection
def test_an_empty_history_is_a_state_not_an_error(store: Store) -> None:
"""Every panel renders on a fresh install, before anybody has typed."""
conn = conn_of(store)
scope = Scope()
assert window(conn, scope) == window(conn, scope)
assert window(conn, scope).events == 0
assert usage.by_command(conn, scope) == []
assert usage.by_person(conn, scope) == []
assert usage.totals(conn, scope)["attempts"] == 0
def test_the_window_says_what_it_covers_and_where_it_came_from(store: Store) -> None:
"""Three days is not a month, and the reader has to be able to tell."""
ran(store, day=DAY, command="cat")
reconstructed(store, day=DAY, hour=9, command="dog")
backfilled(store, day=NEXT, command="pope")
measured = window(conn_of(store), Scope())
assert (measured.first_day, measured.last_day, measured.days) == (DAY, NEXT, 2)
assert (measured.live, measured.journald, measured.daily) == (1, 1, 1)
assert not measured.enough_for_a_trend
def test_counts_include_every_source(store: Store) -> None:
""" "How often was this used" is answered by all three classes of row.
A reconstructed invocation still happened. Only the questions that need
an hour or a duration narrow further.
"""
ran(store, day=DAY, command="cat")
reconstructed(store, day=DAY, hour=9, command="cat")
backfilled(store, day=DAY, command="cat")
(use,) = usage.by_command(conn_of(store), Scope())
assert use.uses == 3
def test_a_group_scope_and_the_roll_up_run_the_same_sql(store: Store) -> None:
"""One query with a nullable parameter, so the two cannot drift."""
ran(store, day=DAY, command="cat", shape=Shape(group_id=GROUP))
ran(store, day=DAY, command="dog", shape=Shape(group_id=OTHER))
everywhere = usage.by_command(conn_of(store), Scope())
here = usage.by_command(conn_of(store), Scope(GROUP))
assert [use.command for use in everywhere] == ["cat", "dog"]
assert [use.command for use in here] == ["cat"]
def test_outcomes_are_counted_apart(store: Store) -> None:
"""Refused is not failed, and neither is a success."""
ran(store, day=DAY, command="cat", minute=1)
ran(store, day=DAY, command="cat", minute=2, shape=Shape(outcome=OUTCOME_COOLDOWN))
ran(
store,
day=DAY,
command="cat",
minute=3,
shape=Shape(outcome=OUTCOME_ERROR, error_class="ValueError"),
)
(use,) = usage.by_command(conn_of(store), Scope())
assert (use.uses, use.ok, use.refused, use.errors) == (3, 1, 1, 1)
assert use.failure_rate == 1 / 3
assert use.refusal_rate == 1 / 3
def test_a_command_nobody_ever_ran_has_no_rates(store: Store) -> None:
"""Dividing by an empty history is a zero, never a crash."""
empty = usage.CommandUse("cat", 0, 0, 0, 0, 0, "", "")
assert empty.failure_rate == 0.0
assert empty.refusal_rate == 0.0
def test_unknown_words_are_not_commands(store: Store) -> None:
"""A word nothing claims belongs to family D, not to the usage counts."""
ran(store, day=DAY, command="cat")
ran(store, day=DAY, command="pulapka", minute=5, shape=Shape(outcome=OUTCOME_UNKNOWN))
assert [use.command for use in usage.by_command(conn_of(store), Scope())] == ["cat"]
assert usage.totals(conn_of(store), Scope())["attempts"] == 1
def test_people_come_back_with_their_favourite(store: Store) -> None:
"""Most active first, each with the command they reach for."""
for minute in range(3):
ran(store, day=DAY, minute=minute, who=ALA, command="pope")
ran(store, day=DAY, minute=9, who=ALA, command="cat")
ran(store, day=DAY, minute=10, who=BASIA, command="dog")
people = usage.by_person(conn_of(store), Scope())
assert [(person.uuid, person.uses, person.favourite) for person in people] == [
(ALA, 4, "pope"),
(BASIA, 1, "dog"),
]
assert people[0].commands == 2
def test_the_matrix_is_every_person_against_every_command(store: Store) -> None:
"""What the management page draws and the group chat never sees."""
ran(store, day=DAY, who=ALA, command="cat")
ran(store, day=DAY, who=BASIA, command="cat", minute=1)
ran(store, day=DAY, who=BASIA, command="dog", minute=2)
assert usage.matrix(conn_of(store), Scope()) == [
(ALA, "cat", 1),
(BASIA, "cat", 1),
(BASIA, "dog", 1),
]
def test_commands_nobody_has_tried_are_named(store: Store) -> None:
"""Which registered commands are dead on arrival.
Takes the registry's list rather than reading it, because which commands
exist depends on the group -- the album's exist only where it does.
"""
ran(store, day=DAY, command="cat")
used = usage.by_command(conn_of(store), Scope())
assert usage.never_used(["cat", "dog", "pope"], used) == ["dog", "pope"]
assert usage.never_used([], used) == []
def test_totals_count_distinct_people_and_commands(store: Store) -> None:
ran(store, day=DAY, who=ALA, command="cat")
ran(store, day=DAY, who=BASIA, command="cat", minute=1)
ran(store, day=DAY, who=BASIA, command="dog", minute=2)
assert usage.totals(conn_of(store), Scope()) == {
"attempts": 3,
"ok": 3,
"people": 2,
"commands": 2,
}