kuhyx/signal-bot
1
0
Fork 0
mirror of https://github.com/kuhyx/signal-bot.git synced 2026-11-13 00:00:48 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Compare commits

base: kuhyx:main
Branches Tags
kuhyx:main
kuhyx:v0.1.5
kuhyx:v0.1.2
kuhyx:v0.1.1
kuhyx:v0.1.0
kuhyx:v0.0.7
kuhyx:v0.0.6
kuhyx:v0.0.5
kuhyx:v0.0.4
kuhyx:v0.0.3
kuhyx:v0.0.2
kuhyx:v0.0.1
kuhyx:v0.0.0
..
compare: kuhyx:v0.0.7
Branches Tags
kuhyx:main
kuhyx:v0.1.5
kuhyx:v0.1.2
kuhyx:v0.1.1
kuhyx:v0.1.0
kuhyx:v0.0.7
kuhyx:v0.0.6
kuhyx:v0.0.5
kuhyx:v0.0.4
kuhyx:v0.0.3
kuhyx:v0.0.2
kuhyx:v0.0.1
kuhyx:v0.0.0

No commits in common. "main" and "v0.0.7" have entirely different histories.
main ... v0.0.7

431 changed files with 1182 additions and 127919 deletions
Show Stats Expand all files Collapse all files

4
.binary-allowlist
View File

@ -1,4 +0,0 @@
# The management page's only non-text asset. An SVG so it stays reviewable
# in a diff: the gate blocks the extension, not the content, and a PNG would
# need a negation in .gitignore as well as an entry here.
web/public/*.svg

48
.env.sample
View File

@ -59,10 +59,6 @@ CAT_API_URL=
DOG_API_URL=
GECKO_API_URL=
COMMONS_API_URL=
# Template; {slug} is substituted per poem.
POEM_TEXT_URL=
# Seconds between liveness probes; lower only for tests.
LIVENESS_TICK_SECONDS=
# Optional. Where !papież keeps the 267 papal portraits it has downloaded.
# Default: $CACHE_DIRECTORY/portraits under systemd (the unit sets
@ -75,52 +71,8 @@ PORTRAIT_CACHE_DIR=
# command without a second account, and it posts for real.
SELFTEST_GROUPS=
# Optional, and private. The album the per-person photo commands draw
# from: a directory holding people.json, tags.json and media/. Everything
# identifying lives in there and nothing about it is in this repository --
# see DOCS-photo-pack.md, and ./web.sh for how it gets filled in.
#
# Leave it empty and the bot looks in $STATE_DIRECTORY/pack (systemd) and
# then $XDG_DATA_HOME/signal-bot/pack. Not finding one is normal: the
# commands simply do not exist. Naming one that is not there, or one whose
# manifest cannot be honoured, fails startup rather than every command.
PHOTO_PACK_DIR=
# Optional. Where the /healthz readiness probe binds.
# Default: 127.0.0.1:8137 -- deliberately not 8000, which unrelated dev
# servers grab, and below the ephemeral range so the kernel cannot take it.
BIND_HOST=
BIND_PORT=
# ---------------------------------------------------------------------------
# scripts/signal_api_watchdog.sh -- the two-minute timer that restarts the
# container when signal-cli can no longer reach Signal. Every value here is
# optional; the defaults are what the timer has always used.
# ---------------------------------------------------------------------------
# What reaches the DESKTOP. journald gets every message either way, twice --
# `journalctl --user -t signal-api-watchdog` and `-u signal-api-watchdog`.
# escalation only the give-up message pops, and only once per window
# always every transition pops, reusing one replaceable slot
# never nothing pops
# Default: escalation. The routine restart/recovery pair is the noise this
# exists to stop -- it used to pop on every firing, with critical urgency and
# no replace id, so one outage left a screen of boxes to dismiss by hand.
WATCHDOG_NOTIFY=
# How long a popup stays up, in milliseconds. Default: 20000.
WATCHDOG_NOTIFY_EXPIRE_MS=
# Consecutive failed probes before the container is restarted. Default: 2.
FAILURES_BEFORE_RESTART=
# Restarts allowed inside RESTART_WINDOW before it gives up and escalates
# instead. A deregistered account is not fixable by restarting, and a
# forever-loop would hide that. Defaults: 3 restarts, 21600 seconds (6h).
MAX_RESTARTS=
RESTART_WINDOW=
# Seconds to wait after a restart before deciding whether it helped, and the
# per-probe curl timeout. Defaults: 45 and 15.
SETTLE_SECONDS=
PROBE_TIMEOUT=

60
.github/workflows/ci.yml vendored
View File

@ -15,37 +15,6 @@ jobs:
steps:
- uses: actions/checkout@v4
# scripts/install_service.sh installs both of these for the host and
# this file never did, so ./scripts/e2e.sh has failed on every commit
# for as long as it has existed: tool/e2e_fixtures.py builds its JPEG
# and MP4 fixtures with ffmpeg, and signal_bot/media_strip.py strips
# metadata with exiftool. Verified rather than trusted, because an
# apt exit code is not the same claim as "the binary resolves".
# Every binary tool/e2e_fixtures.py shells out to, which is the whole
# list rather than the first one that failed: ffmpeg builds the JPEG
# and MP4 fixtures, heif-enc the HEIC one, and exiftool is what
# signal_bot/media_strip.py strips metadata with. scripts/e2e.sh has
# needed all three since it was written and this file installed none of
# them, which is why CI has been red on every commit.
#
# Verified by resolving each binary, not by trusting apt's exit code --
# the same distinction install_service.sh makes for the host.
- name: Install the tools the fixtures are built with
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg libimage-exiftool-perl \
libheif-examples libheif-plugin-x265
for tool in ffmpeg exiftool heif-enc; do
command -v "$tool" || { echo "::error::$tool did not resolve"; exit 1; }
done
# Resolving heif-enc is not the same claim as being able to encode:
# Ubuntu splits the HEVC encoder into its own plugin, so the binary
# is there and answers "No HEVC encoder available". One real encode
# fails here, naming the problem, instead of inside a fixture builder.
ffmpeg -loglevel error -f lavfi -i color=c=red:s=32x32 -frames:v 1 -y /tmp/probe.png
heif-enc -q 50 /tmp/probe.png -o /tmp/probe.heic
test -s /tmp/probe.heic || { echo "::error::heif-enc produced nothing"; exit 1; }
- name: Install uv
uses: astral-sh/setup-uv@v5
@ -67,12 +36,6 @@ jobs:
exit 1
fi
# The name half of this check needs the operator's private data and so
# can only run locally; the shape half -- phone numbers, uuids, Signal
# group ids -- works anywhere and is what CI enforces on every push.
- name: no personal data
run: ./scripts/check_personal_data.sh --all
- name: ruff format
run: .venv/bin/ruff format --check .
@ -85,7 +48,7 @@ jobs:
- name: shellcheck
# --severity=style is the floor CI enforces; the local pre-commit hook
# uses the same flag so a version skew cannot pass locally and fail here.
run: shellcheck -x --severity=style ./*.sh scripts/*.sh
run: shellcheck -x --severity=style run.sh scripts/*.sh
- name: pytest (100% branch coverage)
run: .venv/bin/python -m pytest -q
@ -99,24 +62,3 @@ jobs:
# for eight hours on 2026-08-30.
- name: cold start
run: ./scripts/cold_start.sh
# The management page. Its own job rather than more steps on `check`: it
# needs a Node toolchain that the Python job has no use for, and the two
# halves fail for unrelated reasons.
web:
runs-on: ubuntu-latest
defaults:
run:
working-directory: web
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "24.20.0"
- run: corepack enable
- name: install (exact pins, from the lockfile)
run: pnpm install --frozen-lockfile
- name: tsc --noEmit and eslint
run: pnpm lint
- name: vitest (100% coverage, no excludes)
run: pnpm coverage

38
.gitignore vendored
View File

@ -164,43 +164,5 @@ cython_debug/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
*.jpg
*.jpeg
*.png
*.webp
*.gif
*.mp4
nohup.out
# The private photo album. It lives outside this repository -- in
# ~/.local/share/signal-bot/pack -- and these names exist so that a stray
# copy inside it cannot be committed by accident. people.json and tags.json
# are the two files that hold the names; media/ holds the photos.
people.json
tags.json
media/
pack/
# sources.json is the import ledger. custom.json and adds.json were never
# listed and hold private data too: custom.json is free text somebody typed
# into the group, adds.json holds Signal member uuids.
sources.json
custom.json
adds.json
# The message database is private and never leaves this machine. A live
# SQLite database is three files, so all three are named.
*.sqlite3
*.sqlite3-wal
*.sqlite3-shm
# Wine/Bottles scratch dir, never part of the bot.
bottles/
# The management page's build inputs and reports. dist/ is already covered by
# the bare entry above, which matches at any depth.
node_modules/
web/coverage/
*.tsbuildinfo
# Disposable resume cache for tool/gen_art.py; see commons_api.load_cache.
tool/.commons_cache.json
tool/.commons_cache.tmp

93
.personal-data-allowlist
View File

@ -1,93 +0,0 @@
# Values this public repository is allowed to contain.
#
# Everything a real person could be identified by is refused by
# scripts/check_personal_data.sh, which learns the names from OUTSIDE this
# repository (the photo pack, the message database, .env) so that the list of
# real names is never in it. What is left is shapes -- phone numbers, uuids,
# Signal group ids -- and those need an explicit list of the invented ones the
# documents and fixtures are built from. That list is this file.
#
# One value per line, lower-cased. Add a value here only after checking it
# belongs to nobody.
# Phone numbers. The 555 range is reserved for fiction; the two below are
# structurally valid and deliberately not anybody's.
+15550000000
+48123456789
+420111222333
+48000000000
+48111222333
# UUIDs used as fixtures.
b1c2d3e4-0000-4a00-8a00-000000000001
b1c2d3e4-0000-4a00-8a00-000000000002
# Group ids invented for the offline end-to-end run and the unit tests. Each
# one base64-decodes to a sentence saying it is a fixture.
group.wlrkbexyzghkr05vzldrdc1ncm91cc1pzc0zmi1ienl0zxmhisfd
group.zdjgmfkyagxaqzfuy205mwndmxbaqzb6twkxiwvyumxjeuvositfaelrpt0=
group.wtjllxdhdgnozwqtz3jvdxatawqtmzitynl0zxmhisfd
# The author's own handle, which is the repository owner and appears in the
# project URL, the User-Agent and every copyright line.
kuhy
# The sender name POST /_selftest injects. Not a person.
self-test
# Given names that are also popes. Permitted ONLY in the files that carry the
# papal list, so the same name in a test fixture or a document is still
# refused -- which is exactly how it leaked the first time.
signal_bot/popes.json::marek
signal_bot/popes.json::wiktor
tests/test_versioning.py::wiktor
DOCS-changelog.md::wiktor
# Invented Signal group ids. Every one of these base64-decodes twice to a
# sentence saying what it is, which is how you can tell it belongs to nobody:
# group.WlRKbExYZGhkR05vWldRdFozSnZkWEF0YVdRdE16SXRZbmwwWlhNaElTRT0=
# -> ZTJlLXdhdGNoZWQtZ3JvdXAtaWQtMzItYnl0ZXMhISE= -> e2e-watched-group-id-32-bytes!!!
group.WTI5dVptbG5MV2R5YjNWd0xXbGtMVE15TFdKNWRHVnpJU0VoSVNFaElRPT0=
group.WlRKbExYZGhkR05vWldRdFozSnZkWEF0YVdRdE16SXRZbmwwWlhNaElTRT0=
group.WlRKbExYTmxZMjl1WkMxbmNtOTFjQzFwWkMwek1pMWllWFJsY3lFaElTRT0=
group.WlRKbExYUm9hWEprTFdkeWIzVndMV2xrTFRNeUxXSjVkR1Z6SVNFaElRPT0=
group.WlRKbExXWnZkWEowYUMxbmNtOTFjQzFwWkMwek1pMWllWFJsY3lFaElTRT0=
group.WTI1c1pDMXZibVV0WjNKdmRYQXRhV1F0TXpJdFlubDBaWE1oSVNFaElTRT0=
group.WTI5c1pDMXZibVV0WjNKdmRYQXRhV1F0TXpJdFlubDBaWE1oSVNFaElTRT0=
group.WTI5c1pDMTBkMjh0WjNKdmRYQXRhV1F0TXpJdFlubDBaWE1oSVNFaElTRT0=
group.WTI5c1pDMTBhSEpsWlMxbmNtOTFjQzFwWkMwek1pMWllWFJsY3lFaElTRT0=
group.WTI5c1pDMW1iM1Z5TFdkeWIzVndMV2xrTFRNeUxXSjVkR1Z6SVNFaElTRT0=
group.YjNSb1pYSXRaM0p2ZFhBdGFXUXRNekl0WW5sMFpYTWhJU0VoSVNFPQ==
group.YzJWamIyNWtMV2R5YjNWd0xXbGtMVE15TFdKNWRHVnpJU0VoSVNFaA==
group.ZDJGMFkyaGxaQzFuY205MWNDMXBaQzB6TWkxaWVYUmxjeUVoSVNFaElRPT0=
group.WlRKbExYZGhkR05vWldRdA==
# Invented phone numbers used as fixtures. Reserved / non-allocated ranges.
+15550000001
+15550000002
+15550001111
+15550001234
+15555550123
+15559999999
+48987654321
signal_bot/versioning.py::wiktor
# Given names that are also popes, permitted only in the files that carry or
# assert the papal list. Anywhere else -- a test fixture, a document, a help
# text -- they are still refused, which is the case that leaked.
signal_bot/popes.json::jan
signal_bot/popes.json::marek
signal_bot/popes.json::wiktor
signal_bot/versioning.py::jan
tool/gen_popes.py::jan
tool/gen_popes.py::marek
tool/gen_popes.py::wiktor
tests/test_versioning.py::jan
tests/test_changelog.py::jan
tests/test_popes_command.py::jan
DOCS-changelog.md::jan
# Polish painters whose given name is also somebody's. Permitted ONLY in the
# category list that harvests their work, exactly as a pope's name is
# permitted only in popes.json -- the same name in a test fixture or a
# document is still refused.
tool/art_categories.json::jan

38
.pre-commit-config.yaml
View File

@ -45,37 +45,6 @@ repos:
language: pygrep
types: [python]
# This repository is public and holds a private message database. The
# database is gitignored, but three binary SQLite files reached the
# history anyway before this gate existed, so .gitignore is evidently
# not the whole defence. Exceptions go in .binary-allowlist.
# This repository is public and leaked once: real names, two phone
# numbers and a member's Signal uuid reached GitHub and only came off
# it by deleting the repository. The checker learns what to refuse from
# OUTSIDE the repo -- the photo pack, the message database, .env -- so
# the list of real names is never in the thing being checked.
- id: personal-data
name: no personal data
entry: scripts/check_personal_data.sh
language: system
types: [text]
# The same check on the commit message, because that is where two of
# the leaks actually lived.
- id: personal-data-message
name: no personal data in the commit message
entry: scripts/check_personal_data.sh --message
language: system
stages: [commit-msg]
always_run: true
pass_filenames: true
- id: no-binaries
name: no binary or image files
entry: scripts/check_no_binaries.sh
language: script
pass_filenames: true
# The shared 250-line cap. Checks only the files in the commit, so an
# unrelated commit never breaks. No baseline, no allowlist -- those are
# suppressions. The exemption list lives with the checker in ~/utils so
@ -119,7 +88,7 @@ repos:
entry: .venv/bin/mypy
language: system
pass_filenames: false
args: [signal_bot, fake_signal, tool]
args: [signal_bot, fake_signal]
- id: shellcheck
name: shellcheck
@ -130,14 +99,13 @@ repos:
types: [shell]
# The watchdog is bash, so pytest cannot reach it. Its own suite stubs
# curl, docker, notify-send and logger, and drives every branch --
# including the give-up cap and how many popups a whole window produces.
# curl and docker and drives every branch, including the give-up cap.
- id: watchdog-tests
name: watchdog behaviour (bash)
entry: scripts/test_watchdog.sh
language: system
pass_filenames: false
files: ^scripts/(signal_api_watchdog|test_watchdog|test_watchdog_harness)\.sh$
files: ^scripts/(signal_api_watchdog|test_watchdog)\.sh$
# 100% branch coverage, no omits. Runs on every commit: coverage is the
# whole point of the suite, and a partial run cannot prove it.

173
DOCS-add-photo.md
View File

@ -1,173 +0,0 @@
# Adding a photo from the chat: `!add_photo`
How a member of an album's group proposes a photo for it by sending one,
how the operator approves it, and what keeps that path from leaking anything.
The album itself — where it lives, what its manifests look like, how several
albums coexist — is `DOCS-photo-pack.md`.
Send a photo captioned `!dodaj alfa wakacje2019` in one of the album's groups.
`alfa` resolves to that person's key; `wakacje2019` matches nobody and travels
as a *proposed* label. The photo is stripped and lands in the album's
`pending/`, and the reply is `czeka na zatwierdzenie: 1`. It is **not in the
album yet**: `!alfa` and `!losowe wakacje2019` cannot find it until somebody
approves it.
### Approval: the management page, and nothing else
`./web.sh` shows a **Waiting for approval** section above the tagger whenever
`pending/` is not empty (`web/src/views/pending.tsx`). Each queued photo is a
card with the album's people as checkboxes — the ones the sender named are
pre-ticked — and one checkbox per proposed label. **Approve** moves the file
into `media/`, writes `tags.json`, mints any ticked label in `custom.json`,
records the sender in `adds.json` (with `approved_at`), and asks the bot for
`POST /_reload`. From that moment `!losowe wakacje2019` answers with it, with
no restart. **Reject** deletes the file and its sidecar; nothing else to undo,
because nothing else was written.
The queue is the only door in. The bot's own process never approves anything;
it writes `pending/<content-hash>.<ext>` plus a `.json` sidecar (`keys`,
`labels`, `by`, `at`, `group`) and reads the result back at the reload. The
page can only approve a label the sender actually proposed (anything else is
`400`), and it can keep fewer — an unticked label is simply not minted. If the
bot then refuses the reload (a stale alias, say) the page **reports** it and
does not roll back: the photo is in the album with the tags the approver
chose, and the bot's reason is about something else.
What holds:
- **No typed label travels.** Labels become keys in the registry's
`arg_resolver`, at the dispatcher, *before* the line that logs the command
name; a new word becomes a per-message token (`new1`, `new2`) and is claimed
back by its own handler, so concurrent frames cannot swap labels. The reply
is counts only — `czeka na zatwierdzenie: 2 · 1 już było` — and the proposed
word exists in exactly one place, the sidecar inside the album's `0700`.
`scripts/check_personal_data.sh` reads those sidecars too, so a proposed
name is on the forbidden list before it is ever approved.
- **No cap.** No cooldown, no size limit, no daily limit. `!random` takes the
ordinary pack cooldown; `!add_photo` takes no decorator at all.
- **At most six labels**, the same ceiling `!alfa beta …` has.
- **The attachment must be on the same message**, never a quoted one: a reply
carries the *quoted* message's media in `quote.attachments`, and taking it
would let anyone re-file somebody else's photo.
- **A duplicate changes nothing.** The stored name is a content hash, so a
photo already in the album is `już było`, one already waiting is `już
czeka` — the same photo twice in one message counts as waiting too — and a
mixed batch queues the new files and says how many were which.
- **Approval is whole or not at all.** Media, `tags.json`, `custom.json` and
`adds.json` are written, the album is loaded again, and only a load that
succeeds keeps them. A load that fails rolls all four back and leaves the
photo in the queue with a `400` naming the reason.
- **The second copy goes.** signal-cli keeps its own copy of every inbound
attachment outside the album's `0700`. `!add_photo` deletes it once the file
is queued, and also when the upload was refused.
- **Stripping happens before the wait.** The file in `pending/` is already
the stripped one; `scripts/e2e.sh` re-reads the queue with exiftool before
approving anything, and the album after.
### Stripping: exiftool, with a positive check
`ffmpeg -map_metadata -1 -c copy` was tried first and **fails silently on
photos**: given a JPEG with a real APP1 Exif segment it returned a
byte-identical file, GPS marker intact, and exited 0. `-c copy` passes the
compressed frame through as an opaque packet and a JPEG's Exif lives inside
it. Do not reintroduce ffmpeg for this.
So the gate is a **positive check on the output**, never an exit code:
```sh
exiftool -all= -time:all= -o <dst> <src> # never --overwrite_original
exiftool -G0 -s -all <dst> # then read it back
```
`-time:all=` is **not** redundant with `-all=`. `media_tags.ZEROED_DATE_TAGS`
allows the QuickTime date tags "by value" on the grounds that `-all=` zeroes
rather than deletes them. Measured 2026-08-31: it does not. `-all=` alone left
`QuickTime:CreateDate` at its real value, and the consequence was that **every
mp4 was refused** — the code documented behaviour it did not get. `-time:all=`
is what actually zeroes them.
and every surviving tag must be **structural**. Measured against exiftool
13.55, the plan's original "the surviving set must be empty" would have
rejected every format except JPEG — a valid PNG always reports its IHDR under
`PNG`, a GIF its screen descriptor under `GIF`, an MP4 some fifty `QuickTime`
tags that *are* the container. So the allowlist is per **tag name**
(`signal_bot/media_tags.py`), which keeps the property that matters: an atom
nobody anticipated arrives as a name that is not on the list, and the upload
is refused.
Allowlist, not blocklist, for the reason a naive gate misses: an MP4 geotag is
a QuickTime `udta`/`©xyz` atom reported under **`QuickTime`**, not `GPS`.
### Tracks: ffmpeg, for what no tag allowlist can see
A tag allowlist is structurally blind to one class, and it is the worst one.
`IMG_6924.mp4` in the live album carries two `mebx` timed-metadata tracks, and
after a *complete* `-all= -time:all=` pass its bytes still contain:
```
com.apple.quicktime.detected-face
com.apple.quicktime.detected-face.bounds
com.apple.quicktime.detected-face.face-id
com.apple.quicktime.detected-face.roll-angle / .yaw-angle
```
Per-frame face bounding boxes and a face identity number. exiftool is not
broken for missing them: `mebx` samples live in `mdat` as opaque payload, not
as tags, so **there is no tag to refuse**. The only answer is to drop the
track, which `signal_bot/media_tracks.py` does before exiftool runs:
```sh
ffmpeg -v error -i <src> -map 0:v -map "0:a?" -c copy \
-map_metadata -1 -map_chapters -1 -y <dst>
ffprobe -v error -show_entries stream=codec_type -of csv=p=0 <dst>
```
and every surviving stream must be `video` or `audio`.
**This is not the ffmpeg mistake above.** That one was ffmpeg replacing
exiftool for *tag* removal, where it silently failed on JPEG. Here ffmpeg
removes *tracks*, exiftool still does every tag removal afterwards, and the
gate is a positive read-back of which streams survived rather than an exit
code. `-c copy` is safe for this job precisely because nothing is being
claimed about the frames — the claim is about which tracks exist, and ffprobe
answers that directly. Do not collapse the two jobs back together.
`QuickTime:MetaFormat` is deliberately **absent** from the allowlist: a file
still reporting one has a metadata track the re-mux should have removed, and
refusing it is the correct outcome.
**Both entry points obey this.** `./web.sh --import` used to be a
verbatim `shutil.copy2`; since 2026-08-31 it runs the same stripper, so the
guarantee on this page covers the whole album and not just the chat path.
The list includes the **codec's own configuration** — `CompressorName`,
`VendorID`, and the HEVC parameter set (`HEVCConfigurationVersion`,
`GeneralProfileIDC`, `BitDepthLuma`, …). That was forced by measurement, not
chosen for convenience: with them off the list exiftool 13.55 could produce no
acceptable video *and* no acceptable HEIC, because `CompressorName` lives in
the `avc1` sample description and exiftool reports it as not writable at all
(`Sorry, QuickTime:CompressorName doesn't exist or isn't writable`), and a real
HEIC reports sixteen HEVC config tags. Since HEIC is the iPhone default, the
choice was codec parameters or no phone photos. They describe how the picture
is encoded, never where it was taken or who took it; every location and
identity tag is still refused, and `scripts/e2e.sh` proves a QuickTime GPS atom
does not survive on a file the album actually stored.
Formats are decided by **magic bytes**, never by the filename: jpg, png, gif,
webp, mp4, mov, m4v, heic, heif, avif, webm and svg. Every one of them must
also be in `pack.MEDIA_TYPES`, or a file that uploaded cleanly is refused by
the *next* load and takes startup down.
### ⚠️ .webm and .svg are stored unstripped
exiftool 13.55 can write neither (`exiftool -listwf` lists no `WEBM` and no
`SVG`; a strip attempt returns `Can't write SVG files`). Its SVG *reader* does
not surface `<metadata>` content either, so there is no detection to fall back
on.
kuhy's decision of 2026-08-31 is to keep both on the allowlist and store them
**as they arrive, metadata included**. The consequence, stated plainly: an
`.svg` carrying `<metadata>` or an embedded
`<image xlink:href="data:image/jpeg;base64,…">` keeps whatever is in it, and
every later `!alfa` re-sends that file verbatim. Every other format on the
list is stripped and verified.

163
DOCS-analytics.md
View File

@ -1,163 +0,0 @@
# Command analytics
`message_events` answers *who said how much*. This answers *what they asked
the bot for, when, and whether it worked* — a different table, a different
package, and one that did not exist before 0.2.6.
## Why it needed a new table
The old record was two aggregates, and neither was a history.
- `command_uses` upserts one row per (group, user, command) holding **last use
only**. It is cooldown state.
- `daily_uses` counts uses per day. It is daily-limit state.
Both are written by `store.record_use`, which the dispatcher calls **only on
success**. A gate refusal returned before it and a handler crash returned
before it, so a command that failed every single time was indistinguishable
from one nobody ran. There was no hour, no duration, and no idea which
command a counted message had been.
`command_events` takes one row per **attempt**: outcome, duration, the item
served, and the hour it happened in. `upstream_calls` takes one row per
outbound HTTP request made while serving it.
## The two writes, and why they may never be merged
```
store.commands.record(...) every attempt analytics
store.record_use(...) successes only cooldown + daily limit
```
`CommandRefusedError`'s docstring records the bug that made the split
necessary: recording a refused use slid the cooldown window forward every
time somebody was told to wait, so waiting the stated time never helped.
Now that the history takes refusals, unifying the two calls looks like an
obvious simplification. `test_dispatcher_recording` fires a refused
invocation and asserts both that the history grew **and** that `last_used`
did not move. That test is the whole guard.
## What is never stored
| | |
|---|---|
| the words after a command | `args` holds a registry-resolved pack key (`p1`, `add`, `random`) or `""` / `"+args"`. A user-typed column is the shape that put real names in this repository once. |
| an exception's message | `error_class` is a class name. `str(error)` carries whatever was being processed, and here that is somebody's message. |
| a URL | `upstream_calls.host` is a hostname. A URL carries identifiers, and identifiers belong in `item_id` where a decision governs them. |
| a title or a caption | `item_id` is a QID, a Commons page id, a Wolne Lektury slug or a content hash. An identifier cannot carry a name; free text eventually will. |
## Three kinds of row
`source` says where a row came from, and the three are not interchangeable.
| `source` | group & person | hour | duration | outcome |
|---|---|---|---|---|
| `live` | observed | observed | observed | observed |
| `journald` | observed | observed | **none** | **inferred** |
| `daily_uses` | observed | **none** (`-1`) | **none** | assumed `ok` |
`scope.py` is the only place that names the filters, so "is the exclusion
still there?" is one grep rather than six independent remembers. Hour and
weekday panels take `local_hour >= 0`; latency panels take
`handler_ms IS NOT NULL`; sessions and co-occurrence take a real hour,
because a `daily_uses` timestamp is built from midnight and treating those as
a sequence invents a sitting of forty commands in forty milliseconds.
Durations are `NULL`, never `0`. A zero reads as data and would drag a p50
down — the same lesson `stats/backfill.UNKNOWN_HOUR` already records.
## The backfill
`tool/backfill_analytics.py` recovers the history that predates the table.
Run it against a **copy** first. It is insert-only and re-runnable; there is
no delete-then-rebuild pass, because journald has no retention configuration
and a second run after the log had rotated would delete rows it could no
longer replace.
Two sources that overlap, which is the whole difficulty — summing them would
roughly double every historical count and nothing would fail:
- **journald** knows which command ran and when, but has no group and no
uuid: `dispatcher.py` logs a display name, because the line was written to
be read by a human.
- **daily_uses** knows the group, the uuid and the day, and counts correctly,
but has no times of day at all.
So the join runs **over `message_events`**, which already holds the group,
the uuid, the real hour and the display name of every counted command, and
lacks only the command's name — which is exactly what journald has. Then
`daily_uses` fills the **residual**: `max(0, uses − matched)` per (group,
user, command, day). Nothing is counted twice.
### Two things that were wrong on the first attempt
**The window is symmetric.** The log line is written strictly after the
message arrives, so `[0, +15 s]` looks obviously right. Measured against the
live database: of 299 counted commands, **194 had a journal line earlier than
their own `sent_at_ms`**, by up to 3.2 seconds. The two stamps come from two
different clocks — `sent_at_ms` is set on the *sender's phone*, the journal
stamp is this host's — so their difference is skew and has either sign. The
asymmetric window dropped 47% of the data.
**Matching is a global assignment, not per-row.** Two commands from one
person seconds apart put both log lines inside both windows; picking
best-for-each-row in turn cannot tell them apart and drops both. Closest-pair
first, globally, can: the skew is systematic per sender, so the true pairs are
the two smallest distances available. On the live database that took the
drops from 137 to **zero**.
### One inaccuracy, stated rather than hidden
`dispatcher.py` logs `running command X` **before** the handler runs, so a
journald row proves the command was *entered*, not that it finished. A
handler that then crashed, or a `daily_limit`/`vote_gated` refusal — neither
of which logged anything — is imported as `ok`. Measured on 2026-08-31 the
gap is about 4%. Only `live` rows have outcomes that were actually observed,
and the page says which is which.
## `!analityka`
Text only, Polish, twelve lines as a cap rather than a quota — a line with
nothing to say is dropped, not printed empty. No padding, because Signal
renders a proportional font.
It names at most three people, and only ones whose display name the bot has
seen; everyone else is counted and left unnamed, exactly as the nightly
report does. Full per-person matrices are on the management page. The trend
verdict is suppressed below 14 days: the series is drawn, the direction is
not claimed.
## The Analytics tab
`./web.sh`, then the first tab. Seven read-only routes under
`/api/analytics`, each taking an optional `?group=`. Absent means the
all-groups roll-up, and it runs the *same SQL* — every query is a complete
literal string with `WHERE (:group IS NULL OR group_id = :group)`, so the
roll-up and the per-group answer cannot drift. That spelling is also what
keeps ruff's S608 quiet without a suppression.
Charts are Nivo 0.99.0, always the **fixed-size** components rather than the
`Responsive*` ones: those measure their container, a jsdom container has no
size, and a suite full of charts would otherwise assert against empty divs.
`charts/size.ts` supplies a real measurement in a browser and a stated
fallback everywhere else. Colours are `@kuhyx/web-ui` tokens, not a Nivo
scheme.
## Verifying a change
```sh
.venv/bin/python -m pytest -q # 100%, no omits
./scripts/e2e.sh # incl. analytics_command.yaml
pnpm --dir web lint && pnpm --dir web coverage # 100% on all four metrics
pre-commit run --all-files
# the backfill, against a copy, never the live file
cp signal_bot.sqlite3 /tmp/check.sqlite3
journalctl --user -u signal-bot --no-pager -o short-iso > /tmp/bot.log
python tool/backfill_analytics.py --db /tmp/check.sqlite3 --log /tmp/bot.log --dry-run
```
CI runs `pnpm lint` and `pnpm coverage` but **never `pnpm build`**, so a
bundling or CSP break is invisible to every automated gate. Building and
opening the page is a required manual step, not an optional one.

79
DOCS-changelog-archive.md
View File

@ -1,79 +0,0 @@
# Changelog archive
Releases older than the ten most recent, moved out of `DOCS-changelog.md`
when it crossed the shared 250-line cap. The bot does not read this file:
`signal_bot/changelog.py` only ever loads `DOCS-changelog.md`, and a
backlog announcement is capped at ten releases anyway. Kept because the
release history is the record of what the group was told, and when.
## 0.1.0 — Hyginus
papież nr 9 z 267, pontyfikat 136-140
- podsumowanie liczy teraz zdjęcia i filmy — zdjęcie bez podpisu było
wcześniej warte zero, więc liczby wszystkim podskoczyły
- kto nic nie napisał, pojawia się z zerem
- nowe komendy: !staty, !wczoraj, !tydzień, !miesiąc, !rok, !rekordy,
!korona, !dni, !ja
- bot pamięta historię: rekordy życiowe, serie z koroną, średnie dni
tygodnia. Liczymy od 30 sierpnia 2026
## 0.0.7 — Telesphorus
papież nr 8 z 267, pontyfikat 125-136
- nowa komenda `!papież` (działa też jako `!papiez`, `!papa`, `!pope`) — bot
wrzuca losowego papieża: zdjęcie, imię po łacinie, lata pontyfikatu i link
do polskiej Wikipedii. Do wyboru jest cała lista, 267 papieży od Świętego
Piotra do Leona XIV — czyli ci sami, po których nazywają się wersje bota
## 0.0.6 — Xystus I
papież nr 7 z 267, pontyfikat 115-125
- po restarcie komputera bot wraca sam — wcześniej, jeśli włączył się zanim
Signal był gotowy, poddawał się po 25 sekundach i milczał do końca dnia;
30 sierpnia stało się to na osiem godzin
## 0.0.5 — Alexander I
papież nr 6 z 267, pontyfikat 105-115
- komendy działają teraz niezależnie od tego, na jakiej klawiaturze je piszesz
— `!gekon` napisany rosyjskimi literami (albo jako `!g3ck0`) wreszcie
odpowiada, wcześniej bot go po prostu ignorował
## 0.0.4 — Evaristus
papież nr 5 z 267, pontyfikat 97-105
- `!gekon` pokazuje teraz za każdym razem innego gekona — wcześniej wrzucał
w kółko to samo zdjęcie
## 0.0.3 — Clemens I
papież nr 4 z 267, pontyfikat 88-97
- nowa komenda `!gekon` — bot wrzuca losowe zdjęcie gekona; działa też
`!gecko`, `!geckon`, `!geko`, `!gekony` i `!geckos`
## 0.0.2 — Anacletus
papież nr 3 z 267, pontyfikat 76-88
- bot odpowiada teraz na konkretną wiadomość, cytując ją — w ruchliwej
rozmowie widać, czyje `!cat` to jest
## 0.0.1 — Linus
papież nr 2 z 267, pontyfikat 64-76
- wersje bota nazywają się teraz imionami kolejnych papieży
- po aktualizacji bot ogłasza w grupie nową wersję i listę zmian
- `/healthz` pokazuje wersję i papieża, więc widać co naprawdę działa
## 0.0.0 — Petrus
papież nr 1 z 267, pontyfikat 33-64
- bot taki, jaki był przed drabinką papieską: `!cat`, `!dog` i dzienne podsumowanie

214
DOCS-changelog.md
View File

@ -5,19 +5,6 @@ version number is a pure function of that ordinal — see `signal_bot/versioning
for the two constants and why they are what they are. The short version is that
Jan Paweł II has to land on 2.13.7, and everything else follows.
**The name is Latin, and it is derived, not typed.** `latin_name()` in
`signal_bot/versioning.py` builds it from `popes.json`: the Latin stem plus the
Roman numeral off the Polish name. So 0.0.3 is «Clemens I», not «Klemens I» and
not the raw `latin` field's «Clemens Romanus»; 3.0.4 will be «Leo XIV», not «Leo
Decimus Quartus». Ask the code rather than guessing:
```
python -c 'from signal_bot.versioning import *; print(latin_name(pope_for(14)))'
```
The same string goes in the release commit subject, in `/healthz`, and in the
heading below.
The bot reads this file. The heading format is load-bearing:
```
@ -27,27 +14,7 @@ The bot reads this file. The heading format is load-bearing:
Add the new section at the top when you bump `RELEASE_ORDINAL`. Anything that
is not a `##` heading or a `-` bullet under one is ignored, so prose between
releases is free. The heading *name* is checked rather than trusted —
`tests/test_changelog.py` asserts it equals `latin_name()` for that version —
and the announcement reads `popes.json` regardless, so a typo here fails the
suite instead of reaching a group.
**Preview the announcement before you bump.** Because a bump is a publish,
the announcement was the one feature that could never be exercised first.
`tool/preview_release.py testbot` posts exactly what the next release would
say, into one test group, touching neither `RELEASE_ORDINAL` nor any
`announced_version` row; `--print-only` renders it without sending. It is
fail-closed on `SELFTEST_GROUPS`, so it cannot post a release that does not
exist into the group that would believe it. Write the `##` section below
first -- the preview reads it, so what you see is what ships.
**A release announcement is a photo now.** It carries the pope's portrait, his
Latin name and his pl.wikipedia article, the same three things `!papież` posts.
Only for the pope in the header: a backlog announcement lists up to ten
releases, and ten attachments is the wall the cap exists to avoid. If Commons
will not serve the portrait the announcement still goes out, as text — the
`announced_version` marker is written before the send, so a failure there would
otherwise cost that group its release notes permanently.
releases is free.
**Every release takes the next pope, however small.** A one-line fix consumes
an ordinal exactly like a feature does; the ladder tracks releases, not
@ -71,169 +38,46 @@ this repo: describe what changes for them. 0.0.1 shipped a bullet about
`/healthz`, an endpoint bound to 127.0.0.1 that nobody in a Signal group can
reach, and it was rightly called out.
## 0.2.6 — Stephanus I
## 0.0.7 — Telesfor
papież nr 23 z 267, pontyfikat 254-257
- nowa komenda `!papież` (działa też jako `!papiez`, `!papa`, `!pope`) — bot
wrzuca losowego papieża: zdjęcie, imię po łacinie, lata pontyfikatu i link
do polskiej Wikipedii. Do wyboru jest cała lista, 267 papieży od Świętego
Piotra do Leona XIV — czyli ci sami, po których nazywają się wersje bota
- nowa komenda `!analityka` — kto ile używa bota, o której godzinie i które komendy nikomu się nie przydają
- bot pamięta teraz każde użycie komendy, także to nieudane; wcześniej komenda, która zawsze się psuła, wyglądała jak taka, której nikt nie używa
- historia sprzed tej wersji została odtworzona z dziennika systemowego, więc statystyki zaczynają się od 29 sierpnia, a nie od dzisiaj
- strona zarządzania ma nową zakładkę z wykresami
## 0.0.6 — Sykstus I
## 0.2.5 — Lucius I
- po restarcie komputera bot wraca sam — wcześniej, jeśli włączył się zanim
Signal był gotowy, poddawał się po 25 sekundach i milczał do końca dnia;
30 sierpnia stało się to na osiem godzin
papież nr 22 z 267, pontyfikat 253-254
## 0.0.5 — Aleksander I
- dodano !sztuka — losowe dzieło sztuki, częściej polskie niż nie
- dodano !wiersz — losowy krótki wiersz, w całości, z domeny publicznej
- komendy działają teraz niezależnie od tego, na jakiej klawiaturze je piszesz
— `!gekon` napisany rosyjskimi literami (albo jako `!g3ck0`) wreszcie
odpowiada, wcześniej bot go po prostu ignorował
## 0.2.4 — Cornelius
## 0.0.4 — Ewaryst
papież nr 21 z 267, pontyfikat 251-253
- `!gekon` pokazuje teraz za każdym razem innego gekona — wcześniej wrzucał
w kółko to samo zdjęcie
- dodano !traps
## 0.0.3 — Klemens I
## 0.2.3 — Fabianus
- nowa komenda `!gekon` — bot wrzuca losowe zdjęcie gekona; działa też
`!gecko`, `!geckon`, `!geko`, `!gekony` i `!geckos`
papież nr 20 z 267, pontyfikat 236-250
## 0.0.2 — Anaklet
- jak wpiszecie komendę z literówką, bot podpowie tę właściwą: `!cta`
dostaje odpowiedź „chodziło o `!kot`?”. Wcześniej milczał, więc pomyłka
wyglądała dokładnie tak samo jak bot, który nie działa
- komenda, której bot nie zna i do której nic nie pasuje, dostaje
odpowiedź z podpowiedzią, co z tym zrobić
- nowa komenda `!zaproponuj` — zapisuje pomysł na komendę, której jeszcze
nie ma, np. `!zaproponuj !pogoda pokaż pogodę na jutro`. Pomysły trafiają
na listę, którą czytam
- każde `!słowo`, którego bot nie zna, jest zapisywane. To jest lista
komend, które warto dopisać — więc wpisywanie ich ma sens nawet wtedy,
kiedy jeszcze nie działają
- bez zmian: zwykłe pisanie, samo `!!` i `!kot` z dopiskiem dalej nie
wywołują żadnej odpowiedzi
- bot odpowiada teraz na konkretną wiadomość, cytując ją — w ruchliwej
rozmowie widać, czyje `!cat` to jest
## 0.2.2 — Anterus
## 0.0.1 — Linus
papież nr 19 z 267, pontyfikat 235-236
- wersje bota nazywają się teraz imionami kolejnych papieży
- po aktualizacji bot ogłasza w grupie nową wersję i listę zmian
- `/healthz` pokazuje wersję i papieża, więc widać co naprawdę działa
- album urósł do 180 zdjęć — każda komenda ma ich teraz więcej, a jedna
zaczęła w ogóle działać: nie miała dotąd ani jednego zdjęcia, więc nie
odpowiadała nikomu, kto ją wpisał
- `!dodaj` przyjmuje wreszcie filmy. Do tej pory odrzucał **każdy** film:
bot sprawdzał, czy z pliku zniknęły daty, a narzędzie, którym je usuwał,
wcale ich nie usuwało — więc film nigdy nie przechodził tej kontroli
- z filmów wycinana jest teraz też ścieżka, którą iPhone dopisuje obok
obrazu: ramka po ramce, gdzie na nagraniu są twarze i który to numer
twarzy. Nie da się jej usunąć jak zwykłych metadanych, bo to nie jest
opis pliku tylko osobny strumień — więc film jest przepakowywany i
zostaje w nim wyłącznie obraz i dźwięk
- zdjęcia wrzucane hurtem z folderu przechodzą teraz dokładnie tę samą
kontrolę metadanych co te z czatu — wcześniej były wgrywane jak leci
- to samo zdjęcie wysłane dwa razy, pod dwiema nazwami albo raz z czatu
i raz z folderu, trafia do albumu jeden raz
- to samo co wcześniej: te komendy istnieją tylko w tej grupie
## 0.0.0 — Święty Piotr Apostoł
## 0.2.1 — Pontianus
papież nr 18 z 267, pontyfikat 230-235
- zdjęcia można teraz dorzucać z czatu: podpisz wysyłane zdjęcie
`!dodaj <imię>` i od razu wychodzi na `!<imię>`, bez restartu bota
- można podać kilka etykiet naraz (`!dodaj ktos ktos2`) — zdjęcie dostaje
wszystkie; można też wysłać kilka zdjęć w jednej wiadomości
- etykieta, która nie pasuje do nikogo, zakłada nową — bot mówi wtedy, że
powstała, i sięga się po nią przez `!losowe <etykieta>`
- `!losowe` bez niczego to losowe zdjęcie z całego albumu
- z każdego dodanego zdjęcia bot wycina metadane — w tym miejsce, w którym
zostało zrobione, i model telefonu — po czym sprawdza, czy naprawdę
zniknęły; jak nie potrafi tego zrobić, nie zapisuje zdjęcia w ogóle
- kasuje też swoją kopię odebranego załącznika, żeby nie została poza albumem
- w odpowiedzi są tylko liczby i skróty (`dodano · p1: 4`) — to, co
wpisaliście, nie trafia ani do odpowiedzi, ani do logów
- to samo co wcześniej: te komendy istnieją tylko w tej grupie
## 0.2.0 — Urbanus I
papież nr 17 z 267, pontyfikat 222-230
- nowe komendy ze zdjęciami: każda osoba ma swoją, a kilka naraz
(`!ktos !ktos2`) daje zdjęcie, na którym są wszyscy naraz
- jak takiego zdjęcia nie ma, bot mówi to jednym zdaniem zamiast milczeć
- filmiki też działają — lecą jako filmiki, a nie jako plik, którego nie da
się odtworzyć
- te komendy istnieją tylko w tej jednej grupie: w każdej innej nie ma ich
wcale, `!pomoc` ich tam nie pokazuje i bot na nie nie odpowiada
- zdjęcia i imiona nie są w kodzie bota ani nigdzie w internecie — leżą w
jednym katalogu na komputerze i nigdy go nie opuszczają
## 0.1.7 — Callistus I
papież nr 16 z 267, pontyfikat 217-222
- każda wersja mówi teraz, którym z kolei papieżem jest jej patron i w jakich
latach panował — w ogłoszeniu nowej wersji, w changelogu i przy `!papież`
- `!papież` pisze to samo zdaniem, którym opisuje każdego innego papieża, więc
nie trzeba już zgadywać, czy «Callistus I» to ten piętnasty, czy szesnasty
## 0.1.6 — Zephyrinus
papież nr 15 z 267, pontyfikat 199-217
- statystyki o 21:37 są od teraz o **dzisiejszym** dniu, a nie o wczorajszym —
wieczorny post pokazuje wreszcie ten dzień, który właśnie mijał
- licznik zamyka się o 21:37, więc to, co napiszecie później, wpada do jutra;
post sam to mówi, żeby liczba nie udawała całej doby
- **jedna wiadomość, nie kilka**: dzień, tydzień, miesiąc i to, co bot
przegapił, są blokami w jednym poście, a nie osobnymi powiadomieniami
- dzień, którego bot przegapił (bo go nie było o 21:37), nadal zostanie
wysłany później — nic nie ginie
## 0.1.5 — Victor I
papież nr 14 z 267, pontyfikat 189-199
- ogłoszenie nowej wersji przychodzi teraz ze zdjęciem papieża, po którym ta
wersja się nazywa, i z linkiem do jego strony na polskiej Wikipedii —
dokładnie tak, jak robi to `!papież`
- wersje nazywają się od teraz po łacinie: ta jest «Victor I», a nie
«Wiktor I», a poprzednia była «Eleutherius», nie «Eleuteriusz»
## 0.1.4 — Eleutherius
papież nr 13 z 267, pontyfikat 175-189
- `!pomoc` (albo `!help`) wypisuje wszystkie komendy i mówi jednym zdaniem,
co każda robi — do tej pory nigdzie na czacie nie było ich listy
## 0.1.3 — Soterius
papież nr 12 z 267, pontyfikat 166-175
- "POCZEKAJ 1 SEKUND" odmienia się wreszcie po polsku — teraz jest
1 SEKUNDĘ, 2 SEKUNDY, 5 SEKUND
- odczekanie tego, co bot każe, wreszcie działa: dotąd każda nieudana próba
po cichu przedłużała blokadę, więc kto grzecznie czekał, ten dostawał
odmowę w kółko
## 0.1.2 — Anicetus
papież nr 11 z 267, pontyfikat 155-166
- `!ja` pokazywał wszystkim tę samą godzinę (12:00) — to był artefakt
danych sprzed przebudowy licznika, a nie wasze zwyczaje. Teraz pokazuje
prawdziwe godziny, a komu ich nie znamy, temu nie zmyśla
## 0.1.1 — Pius I
papież nr 10 z 267, pontyfikat 140-155
- komendy mówią teraz, co pokazują — `!korona` na pustych danych tłumaczy,
co się w niej pojawi, zamiast samego "za mało danych"
- `!ja` pisze "między 12:00 a 13:00" zamiast "o 12:00" — bo to godzina, a
nie moment
Releases older than the ten most recent live in **`DOCS-changelog-archive.md`**.
Nothing reads them: `announce` caps a backlog at ten releases, and `since()`
resolves the floor from the version ladder rather than from this file, so a
group that has been quiet for twenty releases still gets the ten newest that
are here. Moved when this file crossed the shared 250-line cap; keep doing
that rather than letting it grow.
- bot taki, jaki był przed drabinką papieską: `!cat`, `!dog` i dzienne podsumowanie

153
DOCS-commands.md
View File

@ -1,153 +0,0 @@
# Commands
The copy that matters is not this file: `!pomoc` prints the same list **in the
group**, in Polish, one line of description per command. That version is built
from the command registry at run time and cannot drift, because
`CommandRegistry.register` takes the description as a required argument and
`help_command.render_help` prints any command its `GROUPS` table forgot to
file anyway, under no heading.
This file is the trigger reference — every spelling that resolves, which
`!pomoc` deliberately does not print.
| Command | Aliases |
|---|---|
| cat | `!kot` `!koty` `!kots` `!cat` `!cats` `!meow` `!miau` + styled variants |
| dog | `!pies` `!psy` `!dog` `!dogs` `!woof` `!szczek` + styled variants |
| gecko | `!gekon` `!gekony` `!geko` `!gecko` `!geckon` `!geckos` |
| art | `!sztuka` `!sztukę` `!sztuki` `!art` |
| poem | `!wiersz` `!wiersze` `!wiersza` `!wierszyk` `!poem` `!poems` `!poezja` `!poezje` `!poetry` |
| trap | `!pułapka` `!pulapka` `!pułapki` `!pulapki` `!pułapkę` `!pulapke` `!trap` `!traps` `!trapy` `!trapper` `!sidła` `!sidla` `!sidło` `!sidlo` `!wnyk` `!wnyki` `!potrzask` `!zasadzka` `!klatka` `!samołówka` `!samolowka` `!beartrap` `!bear_trap` `!snare` `!snares` `!mousetrap` |
| pope | `!papież` `!papiez` `!papieze` `!papieża` `!papa` `!pope` `!popes` `!pontifex` + styled variants |
| stats_today | `!staty` `!statystyki` `!stats` `!dziś` `!dzis` |
| stats_yesterday | `!wczoraj` `!yesterday` |
| stats_week | `!tydzień` `!tydzien` `!week` |
| stats_month | `!miesiąc` `!miesiac` `!month` |
| stats_year | `!rok` `!year` |
| stats_records | `!rekordy` `!rekord` `!records` |
| stats_crown | `!korona` `!crown` |
| stats_weekdays | `!dni` `!days` |
| stats_me | `!ja` `!moje` `!mnie` |
| command_stats | `!analityka` `!command_stats` `!commandstats` `!analytics` `!uzycie` `!użycie` `!ranking` |
| help | `!pomoc` `!pomocy` `!help` `!komendy` `!commands` |
| random | `!losowe` `!losuj` `!random` `!rand` |
| add_photo | `!dodaj` `!dodaj_zdjecie` `!dodajzdjecie` `!add_photo` `!addphoto` |
| suggest | `!zaproponuj` `!propozycja` `!sugestia` `!suggest` `!pomysl` |
| good_bot | `!good bot` `!dobry bot` `!goodbot` `!dobrybot` `!good_bot` `!dobry_bot` |
| food_today | `!co_jadł` `!co_jadl` `!cojadł` `!cojadl` `!jedzenie` `!dieta` + the operator's own in `aliases_custom.json` |
| workout_today | `!co_ćwiczył` `!co_cwiczyl` `!coćwiczył` `!cocwiczyl` `!trening` `!workout` + the operator's own in `aliases_custom.json` |
| mute | `!cisza` `!cicho` `!mute` |
| unmute | `!odcisz` `!wracaj` |
Aliases live in `signal_bot/aliases.json`, not in source, because they are
data: adding one should never mean touching Python. Two constraints on a new
one. It is subtracted from the personal-data gate's word list by
`_repo_vocabulary()`, so it must not be name-shaped; and
`tests/test_nearest.py` fails if it lands within edit distance of a
*different* command's trigger, because the near-miss suggestion below would
then be a coin toss. `!poems` is the one trigger in the table that is only
just legal: it sits two edits from `!pies`, and passes solely because `!poem`
is one edit away and wins the tie. Deleting `!poem` would fail the suite at
`!poems`, nowhere near where the change was made.
The operator can add words of their own without a commit: the Commands tab of
the management page writes `aliases_custom.json` in the bot's state directory
(`CUSTOM_ALIASES_FILE` names another path), keyed by command name, and the
bot appends them to the shipped list when it builds the table. They are
extra spellings only -- `!pomoc` keeps showing the shipped one -- and they go
through the same clash check as `aliases.json`, so a word another command
already answers to is refused, as is a name no command has. That is what
makes the page's save a reload: the bot proves the file before it is kept.
The tagger refuses to give a person a trigger that is already somebody's
custom alias, for the same reason it refuses the shipped ones.
`good_bot` is the one two-word trigger. It works because `fold()` keeps inner
spaces and the registry matches the whole text before it looks at the first
word, so `!good bot` resolves and `!good` alone does not. Two spaces between
the words resolve to nothing, which is what the one-word spellings are for.
It is also the one command that answers with no message: it puts an emoji
on the compliment itself (`POST /v1/reactions/<number>`), drawn from a small
set, and says nothing in the group.
`food_today` and `workout_today` read two logs other programs keep --
diet_guard's `food_log.json` and screen_locker's `log.json`, at those
programs' default paths unless `FOOD_LOG_FILE` / `WORKOUT_LOG_FILE` say
otherwise -- and print the current Warsaw day out of them, fresh on every
call. The spelling the group actually uses for each has a surname in it,
so it is a custom alias in the state directory and never appears in this
repository. Food prints every entry and the day's total, never the daily
budget, which diet_guard keeps on the machine on purpose. Workouts print
every field an entry carries except a reservation phone number, a
screenshot path and the sync record id.
## A word nothing claims
An unrecognised `!word` used to be dropped in silence, which made a typo
indistinguishable from the bot being down. Now it is answered, and kept:
- within one edit (two, from six characters up) of a real trigger, the bot
asks whether that is what you meant — `!cta` → *chodziło o !kot?* The
distance is optimal string alignment rather than `difflib`'s ratio,
because a transposition costs one edit there and shreds the matching
blocks a ratio counts.
- otherwise it points at `!zaproponuj`, which files the idea.
- either way the word and the message go into `unknown_commands`, which the
management page shows. That is the list of commands worth writing next.
Three things stay silent, and each for its own reason. `!kot ładny` is a
real trigger with words after it, and always has been silent. `!!` and `!?`
are punctuation, not a mistyped command. And a command scoped to some *other*
group — the album's, whose triggers are real people's names — is neither
answered nor recorded, because a refusal would announce that the album
exists.
`random` exists **only in the photo album's own groups**, and only on an
installation that has an album at all — everywhere else it is not a command,
not refused and never logged. It takes arguments, and those
arguments are resolved to pseudonym keys by the registry before any handler,
log line or `command_uses` row sees them. See `DOCS-photo-pack.md`.
`mute` (`!cisza [1-10]`) silences the bot in **that group only**, for the
minutes asked or ten by default, and ten at most. It confirms once and then
answers nothing: no commands, no near-miss suggestion, and the nightly report
is held back (not marked, so the next run still says that day). Messages are
still counted. Anybody may ask -- ten quiet minutes is the whole downside.
`unmute` (`!odcisz`) is the one command that runs through a mute, and only
for `TRUSTED_INVITERS`; anyone else is refused without a reply, because a
refusal is a reply. With `TRUSTED_INVITERS` unset, anybody may. There is no
`!unmute` alias: it sits one edit from `!mute`, inside the near-miss budget.
`add_photo` (`!dodaj alfa wakacje` + an attachment) exists in the same groups
as `random` and under the same silence elsewhere. It does **not** put the
photo in the album: it strips it and queues it for approval on the management
page (`./web.sh`), and answers with a count — `czeka na zatwierdzenie: 1`.
Only an approved photo is retrievable, and it is from the moment it is
approved. No cooldown, no size cap, no daily limit. `DOCS-add-photo.md`.
Any command can be **switched off** from the management page (Commands tab),
outright or until a moment, and given its own cooldown. The bot reads the
switch on every attempt, so it is live without a reload; a command that is
off answers `BEEP BOOP !kot WYŁĄCZONE.` (or `… WYŁĄCZONE DO 14:01.`, Warsaw
time) rather than going quiet, and the refusal neither starts the cooldown nor
counts as a use. `!pomoc` still lists it. `DOCS-web-ui.md` has the page, and
the same row is where a custom alias is typed.
## Why `!pomoc` shows the first alias and not a trigger
`Command.triggers` is a `frozenset` of **folded skeletons** — `folding.py`
runs NFKC, strips diacritics and maps Cyrillic/Greek/small-caps/leet
lookalikes onto ASCII, so that `!gecko` typed on a Russian keyboard still
resolves. Everything that makes those triggers robust also makes them
unprintable: the set has no order, `!papież` survives only as `!papiez`, and
the styled variants come out as zalgo.
So `Command.primary` holds the first *raw* trigger instead, which
`aliases.json` lists as the canonical Polish spelling for every command.
Two consequences worth remembering, both from `folding.py`:
- A command name can never contain a digit — `fold` maps `0134578` onto
letters, so `!7dni` would resolve as `!tdni`.
- The styled Unicode aliases in the table above are redundant now; folding
covers them. They are kept because removing an alias can only take
something away.

41
DOCS-latency.md
View File

@ -34,43 +34,10 @@ building anything, measure these two separately:
that is the real win and it makes the prefetch pool far more valuable,
because the pool could hold ids instead of bytes.
## Head-of-line blocking: fixed
This one was confirmed without re-measuring: `bot.py` awaited each frame to
completion, so one slow upstream fetch delayed every message behind it. Now
every frame runs in a task of its own (`signal_bot/inflight.py`), never more
than `LIMIT` at once. The reader waits for a slot *before* spawning, so a
burst applies back-pressure at the websocket instead of fanning out into a
hundred upstream connections. The one thing that must not run side by side
is one person's own commands -- two quick `!kot` would both pass the cooldown
gate before either had recorded a use -- so `signal_bot/user_lock.py`
serialises per sender, and nobody else waits.
## The bot's own share, measured and budgeted
Every measurement above is a measurement of somebody else. To say anything
about *this* code, each served command now reports its timings split into
the bot's own time and everybody else's (`signal_bot/served.py`):
| field | what it covers |
|---|---|
| `queue_ms` | frame read → a slot free: waiting behind other frames |
| `dispatch_ms` | slot → handler: parse, scope, tally, resolve, sender lock |
| `handler_ms` | the handler, wall clock |
| `send_ms` | of which: `POST /v2/send` and friends |
| `upstream_ms` | of which: Commons, the cat API, and the like |
| `own_ms` | `queue + dispatch + handler - send - upstream` |
`POST /_selftest` answers with that block, and `tool/latency.py` holds it to
the budget: **`own_ms` ≤ `OWN_BUDGET_MS` (100 ms) at p95**, over N replies of
a text-only command in a test group, with the command's cooldown switched off
for the run and put back afterwards. Exit 1 when over. The whole reply is
bounded by the container and reported (`round_trip_ms`), not budgeted.
```sh
tool/latency.py testbot # 20 x !pomoc
tool/latency.py testbot --text '!ja' --times 50
```
Head-of-line blocking is a separate, confirmed problem and does not need
re-measuring: `bot.py` awaits each frame to completion, so one slow upstream
fetch delays every other message behind it. Dispatching each frame as its own
task under a bounded semaphore fixes that regardless of what the send costs.
## Rule

27
DOCS-liveness-and-trust.md
View File

@ -29,33 +29,6 @@ Two things now make that impossible to miss:
and a forever-restart loop would hide it. `scripts/install_service.sh`
installs and arms it beside the bot's own unit.
## What reaches the desktop
Every watchdog message goes to journald twice — once via
`logger --tag signal-api-watchdog`, once as the unit's own stdout — and the
desktop popup is a *summary* of that, never the record. `WATCHDOG_NOTIFY`
decides how much of it pops: `escalation` (the default, give-up only),
`always`, or `never`.
It is a summary because it was once a flood. Three things made it one:
- The popups fired per **check**, not per state change, and `give_up()` had no
once-only flag — so a six-hour give-up window announced itself every two
minutes, up to 180 times, while its own text said "no further restarts".
- `alarm()` hardcoded `--urgency=critical`, which `xfce4-notifyd` never
expires, and passed no replace id, so they stacked instead of superseding.
`notify-send --print-id` now feeds `--replace-id` from
`${STATE_DIR}/notify_id`: one slot, reused.
- Worst of all, none of it needed an outage. `scripts/test_watchdog.sh` stubbed
only `curl` and `docker`, so `notify-send` was the real binary and the suite
fired a dozen genuine critical popups per run — from a pre-commit hook. On
2026-08-31 the timer emitted **zero** messages and the desktop still
collected 297, every one of them self-inflicted by committing.
The stubs now live in the harness, and the suite asserts the counts rather than
the absence: a healthy check pops nothing, a routine restart-and-recovery pops
nothing, and a sustained give-up window pops exactly once.
```bash
systemctl --user list-timers signal-api-watchdog.timer
journalctl --user -u signal-api-watchdog.service -n 20

250
DOCS-photo-pack.md
View File

@ -1,250 +0,0 @@
# The photo pack
A pack is a private album the bot posts from: `!alfa` returns a random photo
of alfa, and `!alfa beta` returns one with both of them on it.
**Nothing about a real pack is in this repository, and nothing ever will be.**
Who the people are, what somebody types to get their photo and which file
shows whom live in one directory outside the repo. This document, the tests
and the offline end-to-end run all use the same invented people — alfa, beta,
gamma — so the mechanism can be public while the album stays private.
## Where it lives
```
~/.local/share/signal-bot/pack/
people.json # the commands, and the only place names exist
tags.json # which file shows whom
custom.json # labels approved from !add_photo (optional)
adds.json # who added what, read by nothing (optional)
media/ # the photos and videos themselves
pending/ # !add_photo sends waiting for approval (optional)
```
Found in this order: `PHOTO_PACK_DIR`, then `$STATE_DIRECTORY/pack` (what
systemd hands a **system** unit), then `$XDG_DATA_HOME/signal-bot/pack`.
`StateDirectory=` is deliberately **not** in the unit this repo installs: for
a *user* unit systemd resolves it below `$XDG_STATE_HOME`, which is not where
the album is. Discovery finds the same directory, and leaving `PHOTO_PACK_DIR`
unset keeps a not-yet-filled-in pack from turning `Restart=always` into a loop.
Two absences, deliberately different:
- a pack **nobody named** that is not there, or that has no `people.json`
yet, is an installation without an album: the commands do not exist, and
that is the normal case on every machine and in the whole test suite;
- a pack named in `PHOTO_PACK_DIR` that is missing, or whose manifest cannot
be honoured, **fails startup**. `load_config` raising is an answer, and a
bot that came up healthy and said "no such photo" forever would not be.
### More than one album
`PHOTO_PACK_DIR` takes a **comma-separated list**, the same spelling
`GROUP_IDS` and `TRUSTED_INVITERS` use:
```sh
PHOTO_PACK_DIR=~/.local/share/signal-bot/pack,~/.local/share/signal-bot/pack-testbot
```
Each album carries its own group scope in its own `people.json`, and the two
share nothing but the process: separate people, separate tags, separate
custom labels. That is what lets a test group have working `!add_photo` and
`!losowe` over invented people while the real album does not exist there at
all — which one album cannot do, because its `groups` list *is* the privacy
guarantee.
Trigger collisions between albums are therefore legal. Two albums are two
sets of people and both may call somebody `!alfa`; no message can ever mean
both, because a command a group is not scoped to is not a command there.
`!dodaj` and `!losowe` are registered once per album for exactly this reason.
**Two albums whose groups overlap are refused at startup** with
`DuplicateTriggerError`. Overlapping scopes would make "which album does
`!dodaj` write to?" a coin toss, and a coin toss about which private album
receives a photo is the outcome this whole design exists to prevent.
## people.json
```json
{
"groups": ["group.WlRKbExYZGhkR05vWldRdA=="],
"people": [
{"key": "p1", "triggers": ["!alfa", "!alfka"]},
{"key": "p2", "triggers": ["!beta"]}
],
"combos": [
{"key": "c1", "triggers": ["!oboje"], "all_of": ["p1", "p2"]},
{"key": "c2", "triggers": ["!ktokolwiek"], "any_of": ["p1", "p2"]}
]
}
```
- **`groups` is the whole privacy guarantee.** In any other group these are
not commands at all: not answered, not counted as a command, not named in a
log line. A handler that quietly returned would still leak all three, which
is why the *dispatcher* enforces it — `registry.resolve_with_args` takes the
group and gives back nothing outside it.
- **`key` is a pseudonym**, and the only identifier that reaches the journal
or the `command_uses` table. A command is `pack:p1`; reading either says
somebody asked for a photo and nothing about whose.
- **`all_of` means one photo with all of them; `any_of` means either**, and
where a choice exists the bot picks among the readings that actually have a
photo, so a combination that could be answered is not refused.
- Triggers may not contain a digit — `folding.py` maps `0134578` onto letters,
so `!alfa7` is not what you typed. Two spellings that fold the same are fine
*within one entry* (`!celina`/`!celiną` is one person written twice) and
refused across two.
## tags.json
```json
{"alfa.png": ["p1"], "both.png": ["p1", "p2"], "clip.mp4": ["p1"]}
```
A tag naming a file that is not in `media/` fails startup: a renamed file
would otherwise drop out of the album in silence. An empty list means "seen,
nobody in it", which is how the tagger records a photo you skipped.
Videos work. The attachment goes out as
`data:video/mp4;filename=<name>;base64,<data>` — the spelling is the
container's own, from its `/swagger/doc.json` — because told it is a photo,
Signal renders a clip as a file nobody can play.
## custom.json
```json
{"t1": ["wakacje2019"], "t2": ["gorybieszczady"]}
```
Labels `!add_photo` learned from the chat that match no person. The key is a
pseudonym in the same family as `p1` — it is what reaches `tags.json`, a log
line and a reply — and the label is the only place the typed word is kept.
Custom tags define **no commands**: `!random <label>` is the only door to one.
That is deliberate, and it is what makes a live album swap safe — a swap
replaces pack *data* only, and can never register a trigger or widen a
command's groups.
A label that folds onto a person's trigger, or onto another key's label, fails
startup rather than shadowing the command it collides with.
## adds.json
```json
{"a3ae52c7….jpg": {"by": "<uuid>", "at": 1756600000, "approved_at": 1756700000}}
```
Who added which file, and when it was approved. **Nothing reads it** —
`pack_load` ignores it — and it exists only so `!undo_photo` can be built
later without a gap in its history; the adder is recorded at add time or not
at all. It lives inside the album (`0700`, outside this repository), so it is
handled exactly like the photos.
## sources.json
```json
{
"IMG_0234.jpg": {"source": "9f2c…", "stored": "b71e…"},
"a3ae52c7….jpg": {"source": null, "stored": "a3ae…"}
}
```
The import ledger: every file the album has ever taken in, under **two**
SHA-256s, because stripping changes the bytes.
- `source` — the file as it *arrived*, before metadata was removed. This
answers "have I imported this photo before?" when the same export folder is
pointed at the album a second time.
- `stored` — the bytes on disk now. This answers "is the photo somebody just
uploaded from the chat already here?", because `!add_photo` hashes *after*
stripping.
Neither alone is enough. The same photo arriving through `--import` and
through Signal has different source bytes — the phone re-encodes — but strips
to the same output. `source` is `null` when the original bytes were never
kept, which is exactly the `!add_photo` files.
Like `adds.json`, **`pack_load` never touches it.** A corrupt ledger degrades
to "I have seen nothing", costing a duplicate photo; a corrupt ledger checked
at startup would be a restart loop under `Restart=always`.
## Adding from the chat: !add_photo
A member of one of the album's groups sends a photo captioned
`!add_photo alfa wakacje`. It is stripped and lands in **`pending/`** — under
its content hash, beside a `<file>.json` sidecar with the proposed `keys`,
`labels`, sender and group — until the operator approves it on `./web.sh`,
which is when it is tagged, moved into `media/` and retrievable. `pack_load`
never reads `pending/`. The path has its own document, because its metadata
handling and privacy invariants are longer than the album format itself:
**`DOCS-add-photo.md`**.
## Filling it in: ./web.sh
```sh
./web.sh --import ~/photos # dedup, strip, import, then tag
./web.sh --import ~/photos --dry-run # the report only
./web.sh --untagged # only items with no tags yet
./web.sh # carry on where you left off
```
`tag.sh` sets up `.venv` on first use, so this works on a fresh clone with
nothing installed. Same for `run.sh`, which used to print the two commands to
type and exit 1.
It serves one page on `127.0.0.1:8140`: the item, a labelled button per
person, `1`–`0` then `q`–`p` to toggle, `→`/`←` to move. Every toggle rewrites
`tags.json` atomically, so closing the tab loses nothing. Names are typed
**into the page**, never as arguments — an argument would land in the shell
history — and the group is picked from the live `/v1/groups` listing rather
than pasted.
**`--import` is not a copy.** Every file is identified by its magic bytes
(not its suffix), stripped through the same `media_strip` path `!add_photo`
uses, and checked against `sources.json` twice: the source hash first, then
the post-strip hash. What the album refuses is **moved, never deleted**, to
`_skipped/<reason>/` beside the source folder, and the printed report has to
account for every file seen:
```
174 files seen
1 not media -> _skipped/not-media/
0 could not be stripped -> _skipped/unstrippable/
82 already in pack -> _skipped/already-in-pack/
24 duplicate in batch -> _skipped/exact-dup/
67 imported
---
174 accounted for 85 -> 152 in media/
```
Every bucket prints even at zero: a suppressed zero is a silently missing
category, and `0 could not be stripped` is how you see the video pipeline is
still working. Re-running on the same folder is a no-op — that is what the
ledger is for.
**`--untagged` filters the page, never the server's list.** `/media/{index}`
and `/api/tag` both index the full album, so a shorter list on the server
would show item *i* of the album while tagging item *i* of the filtered set —
every photo tagged with the wrong people, and no error anywhere.
**Adding a person works mid-session.** The roster used to be write-once: the
setup form only appeared on an empty album, `Album.setup` overwrote the file
whole, and keys were positional (`p{len+1}`), so inserting somebody renumbered
everyone after them and repointed every `tags.json` entry at the wrong person.
Now keys come from the highest number in use, the form merges, and there is an
"add person" box in the tagger itself — for the photo of somebody new that is
already on the screen.
When you stop it, it prints how many items were never opened and how many
photos each command would have. **A command showing `0` is dead on arrival**;
that is the number to check before restarting the bot, not after somebody in
the group tries it.
## Running the mechanism without an album
`scripts/e2e.sh` writes a synthetic pack with `tool/e2e_pack.py` and proves,
offline and with no Signal account, that one name posts a photo, two names
intersect, an impossible combination gets one line and no attachment, a video
leaves as a video, and the same command in another group produces **zero**
sends. It also drives `!dodaj` against the real exiftool on media generated at
run time (no photo enters this repository), approves the queue through the
management page it boots beside the bot, and has `!losowe` answer, no restart.

116
DOCS-privacy.md
View File

@ -1,116 +0,0 @@
# What is private, and what stops it leaking
This repository is public. The bot it describes reads a private group chat,
posts from a private photo album, and keeps a database of who said how much.
None of that may be in here, and on 2026-08-30 some of it was.
## What leaked, and how
Real first names, one full name, two phone numbers and a group member's
Signal uuid were in the tree *and* in the public history. They arrived one
way every time: **somebody reached for a real name because it was the example
in front of them** — a test fixture built from a live database dump, an
envelope pasted verbatim out of a running container, a help text written
while looking at the actual group.
Fixing it needed the tree scrubbed, all 56 commits rewritten with
`git filter-repo`, and then the repository **deleted and recreated**, because
GitHub keeps orphaned commits fetchable by their SHA after a force-push and
only the repository going away takes them with it. The closed pull requests
and the star went too.
## What is private, and where it lives
| | Where | Never in git because |
|---|---|---|
| the photos and who is in them | `~/.local/share/signal-bot/pack/` | it is outside the repo, and `.gitignore` also names `people.json`, `tags.json`, `media/`, `pack/` and every media extension |
| the message database | `signal_bot.sqlite3` | `*.sqlite3` and its `-wal`/`-shm` siblings are ignored |
| the account's number, groups, inviters | `.env` | `.env` is ignored; `.env.sample` carries the shape and no values |
| everyone's names | the database and the pack manifest | nothing reads them into the repo; the bot logs `pack:p1`, never a name |
| every command anybody has run | `command_events` and `upstream_calls` in the database | the database is ignored, and the columns are chosen so there is nothing to leak: a resolved pack key or `+args` instead of the words typed, an exception's *class* instead of its message, a hostname instead of a URL, and a public identifier instead of a title. See `DOCS-analytics.md`. |
| what people typed after a `!` the bot did not know | `unknown_commands` in the database | the database is ignored; the log line names neither the word nor the message, and the export the management page writes goes to the state directory, never into this tree |
All of it is `700`/`600` on disk. The repository never needs any of it: the
tests, the documents and the offline end-to-end run use invented people —
alfa, beta, gamma, ala, basia, celina — and invented ids that base64-decode
twice into a sentence saying what they are.
## The gate
`scripts/check_personal_data.sh` refuses a commit that carries personal data.
It runs on staged files, on the commit message, and over the whole tree in
CI. Two halves, because they fail differently:
- **Names cannot be listed here** — a file of real names kept in the repo to
check against *is* the leak. So the checker learns them at run time from
outside: the photo pack's manifest, the message database, and `.env`. It
never prints the word it matched, because this output reaches CI logs.
- **Shapes can be listed** — a phone number, a uuid and a Signal group id
look like themselves anywhere, so those are matched by pattern and compared
against `.personal-data-allowlist`, which holds only invented values.
On a machine with none of that private data (CI, a fresh clone) the name half
has nothing to compare against and **says so** rather than passing quietly.
It found a real leak the first time it was run: the live test group's id, in
two spellings, in `tests/test_config.py` — which four rounds of grepping by
hand had missed, because nobody thinks to grep for base64.
## When it fires
```
tests/test_thing.py:12: a name from your private data
```
Use an invented name. `ala`, `basia`, `celina`, `darek` are already in use;
so are `+15550000000` and ids that decode to `e2e-watched-group-id-32-bytes!!!`.
```
DOCS-thing.md:40: phone number '+48123456789' is not in .personal-data-allowlist
```
Either it belongs to somebody — replace it — or it is invented, in which case
add it to `.personal-data-allowlist` **after checking that it is**. A line
there may be scoped to one file (`signal_bot/popes.json::hadrian`), which is how
a given name that is also a pope stays refused in every test fixture that has
no business carrying it.
## Why no command commits the words it posts
Every content command posts what it just fetched. `commons.caption_for` says
why that matters: *the text is fetched at run time and posted to the group;
none of it is ever written into this repository, which is why it may be free
text where `traps.json` may not.*
`!wiersz` was written the other way first, and it is worth recording because
the other way looked entirely reasonable. A pack of several hundred short
public-domain Polish poems was harvested, filtered for names, and committed —
and `scripts/check_personal_data.sh` refused it. Three lines carried a real
group member's given name. After a further filter that discarded 31% of the
corpus, two still did.
The reason is structural rather than a matter of filtering harder. Polish
verse capitalises **every** line, so a capital at the start of a line carries
no information — and that is exactly where a name in a poem tends to sit, in a
dedication (`Basi`), an epigraph credit (`Shakspeare.`) or simply mid-sentence
across a line break. A filter strict enough to catch those rejects all verse.
So the pack holds Wolne Lektury **slugs**, and the poem, its title and its
author arrive over the wire when somebody types the command. `Maria
Konopnicka` is posted into the group every week and appears nowhere in git.
The same shape as `traps.json`'s page ids and `art.json`'s, for the same
reason: an identifier cannot carry a name, and free text eventually will.
The failure mode this avoids is the nasty one — it appears **only locally**.
CI has no private data to compare against, so CI stays green while every
commit on the maintainer's machine is refused, over data nobody can edit out
by hand.
## What the gate cannot do
- It does not scan history. Rewriting that is a deliberate, separate act.
- It knows only what is on *this* machine. A name nobody has ever typed into
the group is a name it cannot recognise.
- It does not stop the bot's own logs holding display names —
`journalctl --user -u signal-bot` does, and `--vacuum-time=` is how that is
emptied.

74
DOCS-reload.md
View File

@ -1,74 +0,0 @@
# Picking up a change without a restart
Two kinds of change reach the running bot, and they are handled differently.
## Data: `POST /_reload`
Aliases (shipped, and the operator's own `aliases_custom.json`), an album's
`people.json`/`tags.json`, and the `.env` and album settings the management
UI writes are *data*. The command table is built from that data by one
function (`signal_bot/wiring.py`), and `RegistryHolder` can run it again:
the new table is built whole, then swapped in with a single assignment. Nothing
restarts, the websocket stays up, and no frame is lost or replayed.
```sh
scripts/reload.sh # rebuild the table, report how many commands are live
scripts/reload.sh '!nowaosoba' # ...and then type that at the bot in testbot
```
The answer is what the bot is serving **now**:
| status | code | meaning |
|------------|------|--------------------------------------------------------------------|
| `reloaded` | 200 | the new table is live; `commands` is its size |
| `kept` | 409 | the data no longer builds; the previous table is still answering, and `reason` says what was wrong |
| `connecting` | 503 | the bot has not reached Signal yet; there is no table to rebuild |
A rejected reload is deliberate: the builder reads files people edit by hand
and pages write mid-session, and the only safe answer to a file that does not
load is the table that was already answering. Only bad *data* is survived
(`ConfigError`, `DuplicateTriggerError`); a programming error still surfaces
as a 500 and a traceback in the journal, because a bug is not a file.
Startup is the one time a bad table is fatal: there is nothing to fall back
on, and `Restart=always` looping on a `ConfigError` is a clearer signal than a
bot that answers nothing.
One thing the management UI writes needs neither: the per-command switches
on its Commands tab (`/_commands`) live in the bot's own database and are read
on every attempt, so they are live on the next message without a rebuild.
They are also what a reload does *not* reset -- `command_settings` is a
permanent table. The aliases field on the same tab is the opposite case: it
is data, so the page performs the reload itself after writing the file, and
undoes the write if the answer is not `reloaded` (`DOCS-web-ui.md`).
## Code: `scripts/reload.sh --code`
A Python change needs the process replaced. The restart is about a second and
lossless (signal-cli replays what arrived meanwhile), but it is a deploy step,
so it is the flag rather than the default:
```sh
scripts/reload.sh --code '!pomoc'
```
restarts the unit, waits for `/healthz` to report `ok` again, prints the
version it came back as, and -- with a command given -- types it in the test
group so "restarted" means "answered". There is no hot-reload of Python on
purpose: `importlib.reload` leaves every object the old module already
created bound to the old classes, which is a bug that looks like a feature
until the first `isinstance` check.
## Proven where
`scripts/e2e.sh` brackets a `scripts/reload.sh` with the two scenarios in
`fake_signal/proofs/`: `!delta` is unknown, a fifth person is tagged into the
synthetic pack, the script runs, and the same process posts delta's photo.
The route itself is covered in `tests/test_app.py`, the holder in
`tests/test_registry_holder.py`, and the album case in `tests/test_bot.py`.
The switches get the same bracket: `switch_off.yaml` and `switch_on.yaml`
around a `PUT /_commands/cat`, with no reload in between. The operator's
aliases get a three-way one: `alias_before.yaml` (`!e2ekot` is nobody), the
file is written and `scripts/reload.sh` runs, `alias_after.yaml` (it posts a
cat), then a file naming a command that does not exist is refused with
`kept` and `alias_kept.yaml` shows the previous table still answering.

9
DOCS-selftest.md
View File

@ -29,15 +29,6 @@ scripts/selftest.sh testbot '!papież' 3
the self-test is always the same author, so a tighter loop just exercises the
cooldown instead of the command.
## What it answers with
`delivered`, the group id, the text -- and, when the text was a command,
`served`: what that one reply cost, split into the bot's own time and the
time spent on signal-cli and upstream (`signal_bot/served.py`, and the table
in `DOCS-latency.md`). A plain message or an unknown word is delivered and
counted but has no `served`, because nothing ran. `tool/latency.py` reads
that block repeatedly to hold the bot to its budget.
## What it will not do
It **posts for real**, into a real group. Two things keep that from becoming a

83
DOCS-signal-envelopes.md
View File

@ -28,87 +28,6 @@ Reference for the two JSON frames `signal-cli-rest-api` pushes down
}
```
## A message with an attachment
Captured from the live API on 2026-08-30 with `LOG_LEVEL=DEBUG`, which makes
the dispatcher log each frame's shape (keys and types, never values -- see
`signal_bot/envelope_shape.py`). This is a caption-less PNG posted in the real
group, reproduced exactly as the logger described it:
```
{account: str, envelope: {
dataMessage: {
attachments: [{caption: null, contentType: 'image/png', filename: null,
height: int, id: str, size: int, uploadTimestamp: int,
width: int} x1],
expiresInSeconds: int,
groupInfo: {groupId: str, groupName: str, revision: int, type: 'DELIVER'},
isExpirationUpdate: bool,
message: null,
timestamp: int,
viewOnce: bool},
serverDeliveredTimestamp: int, serverReceivedTimestamp: int,
source: str, sourceDevice: int, sourceName: str,
sourceNumber: str, sourceUuid: str, timestamp: int}}
```
Three things this settles, none of which were guessable:
- `attachments` is a list of objects and the type lives in `contentType`. That
is what `models._attachments` reads and what `counting.media_kind` maps.
- **A caption-less photo sends `message: null`**, not `""` and not an absent
key. Before 0.1.0 `should_count` ended in `bool(message.text)`, so every one
of these was silently worth nothing.
- Read them off the *resolved* payload from `_payload()`, never off
`envelope["dataMessage"]` -- see "The trap this caused" below.
Fields nothing reads yet, recorded so the next person does not have to capture
them again: `expiresInSeconds`, `isExpirationUpdate`, `viewOnce`,
`groupInfo.groupName`, `groupInfo.revision`, and the attachment's `caption`,
`filename`, `width`, `height`, `size` and `uploadTimestamp`.
**Open question, deliberately not assumed.** `caption` is a field on the
attachment, and it was `null` here because this photo had none. Whether a
photo *with* a caption puts the text in `message` or in `attachments[].caption`
has not been observed yet. `char_len` currently measures `message` only, so if
it turns out to be the latter, captioned photos are recording a length of 0.
Leave `LOG_LEVEL=DEBUG` on and watch for `caption: str` to settle it.
## A reply carries `quote` -- with its own attachments
Also captured live, on 2026-08-30. This is a *text* reply to somebody else's
photo:
```
dataMessage: {
expiresInSeconds: int, isExpirationUpdate: bool, viewOnce: bool,
groupInfo: {groupId: str, groupName: str, revision: int, type: 'DELIVER'},
message: str,
quote: {
attachments: [{contentType: 'image/jpeg', filename: str,
thumbnail: {caption: null, contentType: 'image/jpeg',
filename: str, height: int, id: str, size: int,
uploadTimestamp: int, width: int}} x1],
author: str, authorNumber: str, authorUuid: str, id: int, text: str},
timestamp: int}
```
**`quote` has its own `attachments` array**, holding the *quoted* message's
media. There is no top-level `attachments` here at all, because the replier
sent only text.
So anything looking for attachments must read them off the top level of the
resolved payload and nowhere else. Searching the payload for an `attachments`
key would credit the replier with a photo somebody else posted -- and in a
group where people reply to photos constantly, that inflates one number
without ever looking wrong. It is the same wrong-nesting-level mistake that
once made the bot ignore its own stickers.
`quote.text` is the quoted message's body, and is not read for the same
reason: it is somebody else's message, already counted when they sent it.
`tests/test_models.py::test_a_quoted_photo_is_not_the_replier_s_photo` pins
this against the exact frame above.
## Somebody changed the membership: `groupInfo.type` is `UPDATE`
The only frame that names *who* added the bot to a group, which is what
@ -128,7 +47,7 @@ who made the change. Captured live on 2026-08-29:
"dataMessage": {
"timestamp": 1788006851503,
"message": null,
"groupInfo": { "groupId": "Y29uZmlnLWdy…", "groupName": "testbot", "type": "UPDATE" }
"groupInfo": { "groupId": "Y29uZmlnLWdy/…", "groupName": "testbot", "type": "UPDATE" }
}
},
"account": "+420111222333"

102
DOCS-statistics.md
View File

@ -1,102 +0,0 @@
# Statistics
This is the **message** history: who said how much, when. The **command**
history -- what people asked the bot for, whether it worked and how long it
took -- is a different table and a different package; see
`DOCS-analytics.md`.
## What is recorded
One row per message in `message_events`: sender uuid and the display name they
had at the time, the Signal send timestamp, the Warsaw day and hour, the media
kind and count, the character length, and whether it was a command or a reply.
**Never the message body.** `tests/test_stats_events.py` asserts the column set
exactly, so a later edit cannot quietly add one. Reactions are not messages and
stay out. The bot's own posts never count — decided by which envelope branch
carried the payload, not by comparing identities, because the account is
spelled as a uuid on some frames and a +E.164 number on others.
A photo posted without a caption counts. Before 0.1.0 it was worth zero, which
is why everyone's numbers jumped on release.
## Days
A day is a calendar day in **Europe/Warsaw**, computed when the event is
recorded and stored as a string. Every query groups on that string rather than
on a range of timestamps, because Warsaw days are not all 24 hours long —
2026-03-29 is 23 and 2026-10-25 is 25 — and a timestamp range moves the
boundary twice a year.
Two consequences worth knowing before editing anything here:
- "Yesterday" is `date - timedelta(days=1)` on the *date*. Subtracting 86400
seconds lands at 23:00 the previous evening on a short day.
- Elapsed-time arithmetic must convert to UTC first. Subtracting two aware
datetimes that share a tzinfo gives the *wall-clock* difference, so the naive
spelling reports every day as exactly 24 hours and hides the bug.
## The nightly report
Posted at `REPORT_HOUR:REPORT_MINUTE` for the previous full day: the table,
members who said nothing at 0, a summary line, and up to three highlights
chosen from around twenty candidates — one per family, so a record, a share
and a crown streak about the same person do not become three lines.
Capped at twelve lines. Names are not padded: Signal renders a proportional
font, so `ljust` lines up in a terminal and on nobody's phone.
Only people whose display name has been seen are named. Somebody who has never
written is counted, not named — the group listing knows them only as a raw
uuid or a phone number, and printing either into a chat is a leak.
Seven numbers are funny to this group and get a marker wherever they appear
as a *count* — `69 (nice)`, `2137 🙏`, `1410 ⚔️`, `1916 🇵🇱`, `420 🌿`,
`314 🥧`, `67 🤷` — in the nightly report, every `!stats_*` and `!ja`, and the
highlights. Percentages, shares and the `(N 📷)` aside stay bare: a 69% share
is not a 69, and marking the photo count next to an already-marked message
count would be two jokes on one line. The table is `FUNNY` in `stats/text.py`.
## Outages
Nothing is ever deleted, and the last day reported is recorded per group.
- **Missed counts** come back on their own. An event carries Signal's own send
timestamp, signal-cli keeps receiving in json-rpc mode while the bot is
down, and the replay on reconnect is absorbed by the `(group, uuid,
sent_at_ms)` primary key. Messages sent during an outage are filed under the
day they were actually sent.
- **Missed reports** are posted when the bot returns, oldest first, bounded at
a week. A day nobody spoke on is marked settled rather than reconsidered
forever; a day whose *send* failed is left unmarked so the next run retries
it.
## The backfilled day
2026-08-30 in the real group is carried over from the pre-0.1.0 counter, which
had no timestamps. Its totals are real; its hours and lengths are not, and its
window was 22 hours rather than a day. It is marked in `group_meta`, claims no
average message length, and is excluded from every record and average — a
merged day would otherwise hand somebody a personal best they never set.
The counter could not be split across the two days it spanned. Four places
were checked for a real inbound timestamp: journald (commands only),
signal-cli's `account.db` (no inbound table), Signal Desktop (unlinked, and it
does not backfill on link), and `signal-api/attachments` (dated, but media
only and with no sender). The attachment mtimes put 12 files on 29 August
against 79 on the 30th inside that window, which is why the whole counter sits
on the 30th.
## Debugging the wire format
`LOG_LEVEL=DEBUG` makes the dispatcher log each frame's *shape* — keys and
value types, never values, with a small allowlist for MIME and groupInfo types
that are format rather than content. Libraries stay at WARNING whatever the
bot is set to.
journalctl --user -u signal-bot -f | grep 'envelope shape'
Captured shapes live in `DOCS-signal-envelopes.md`. Read that before changing
anything that parses an envelope: `quote` carries its own nested `attachments`
array belonging to the *quoted* message, and counting it would credit a
replier with somebody else's photo.

250
DOCS-web-ui.md
View File

@ -1,250 +0,0 @@
# The management page
`./web.sh` serves one page on `127.0.0.1:8140`: whether the bot is alive, what
it is configured with, and the photo album's tagger. It replaces
`tool/tagger.py`, which served the album half of this on the same port.
```sh
./web.sh # the configured pack
./web.sh --dir ~/pack --port 8140
./web.sh --import ~/photos # dedup, strip and import, then serve
./web.sh --import ~/photos --dry-run # the report, changing nothing
./web.sh --untagged # only items with no tags yet
```
## Why it is started by hand
A systemd unit would be more convenient and is the wrong shape. This page can
rewrite `.env` — the file that decides which groups the bot answers in and who
may add it to one — and a unit would keep that writable on loopback all day
for the sake of a bookmark. Started by hand, the window is the few minutes
somebody is actually looking at it.
It is also a **second process**, not more routes on the bot. The bot's own
port already serves `POST /_selftest`, which posts for real into a real group;
putting a configuration writer beside it would put both behind one mistake.
## Why loopback is not the whole defence
Binding to `127.0.0.1` stops the network. It does not stop the browser already
running on this machine, and two attacks reach a loopback port from an
ordinary web page. `signal_bot/manage/guard.py` closes both, in about forty
lines, with nothing to configure and no session to keep:
- **Cross-origin form posts.** Any site can POST a `<form>` at
`127.0.0.1:8140` without reading the reply. A form can only send
`application/x-www-form-urlencoded`, `multipart/form-data` or `text/plain`,
so every mutating request must declare `application/json` — which a form
cannot. Anything else is **415**.
- **DNS rebinding.** A hostname the attacker controls is re-pointed at
`127.0.0.1` after the page loads, which makes the browser treat this app as
same-origin. The rebound request still carries the attacker's hostname, so
a `Host` that is not loopback is **421**.
Neither is authentication, and neither pretends to be. They are the two
cheapest things that make "only this machine" mean it.
## Editing .env
Three refusals, in this order, and the order is the point.
1. **A key that may not be written is refused before anything is parsed**, so
a read-only key cannot be smuggled past by making the rest of the file
invalid. `403`, naming the key.
2. **The whole candidate file is validated by the function startup uses** —
`load_config(candidate, use_dotenv=False)`. An edit this bot would refuse
at boot is refused at the form instead, with the journal's own sentence.
`400`.
3. **Only then is anything written.** A rejected edit leaves `.env`
byte-identical.
The write itself is line-by-line: only the `KEY=` line of a key that actually
changed is replaced, so the paragraphs explaining each setting survive an edit
made in a browser. A key the file never had is appended under a marker rather
than dropped among comments that explain something else. Temp-then-replace
with mode `0600` reapplied before the swap, because this file holds the
account's number and is edited from a tab that can be closed mid-save.
### What is read-only, and why
| Key | Why the page will not write it |
|---|---|
| `PHONE_NUMBER` | Identifies the account; the container holds the matching registration. |
| `API_BASE_URL` | Repointing it sends every outgoing message and attachment fetch elsewhere at the next restart. Strictly worse than changing the number. |
| `DATABASE_PATH` | Repointing it silently orphans the whole statistical history. |
| `PHOTO_PACK_DIR` | **Cannot be validated.** `_photo_pack_dirs()` reads `os.environ` directly rather than the mapping it is handed — deliberately, since that is what keeps the test suite away from a real album. |
| `BIND_HOST` / `BIND_PORT` | Moving the bot's port from a page that proxies to it. |
### Group settings need a group listing
`GROUP_IDS` and `SELFTEST_GROUPS` are editable only while `/api/groups` has
actually returned something. `parse_group_ids` checks *syntax*; startup also
requires each listed group to be one the bot really joined, judged against a
listing that was really read. With the container down, a syntactically valid
write can name a group the next startup rejects — a scoping mistake turned
into a unit that will not start. The page says so and leaves the rest editable.
Group ids are **picked, never typed** — `GROUP_IDS` and `SELFTEST_GROUPS`
render as a `<select>` filled from `/api/groups`, for the same reason the
tagger picked them: an id is 44 characters of base64.
> `SELFTEST_GROUPS` is editable and worth understanding before you widen it.
> It is what keeps `POST /_selftest` fail-closed; every group named in it is a
> group the bot can be made to post arbitrary text into.
## The album, in two halves
Which half you get is the album's own `configured` flag, as the tagger decided.
- **No `people.json` yet** → the roster form. One person per line, aliases
separated by commas; a line with an `=` is a combination, `+` for "both of
them" and `|` for "either of them". Keys are reused for anyone already on
the roster and otherwise handed out one past the highest in use — never the
count, or a key deleted by hand would be reissued and inherit somebody
else's photos (`pack_people.next_key` is the authority). Names are typed
into the page, never passed as arguments: an argument lands in shell history.
- **Configured** → the tagger: the item, a checkbox per person, saved on the
click rather than on a "done" button, and an "add somebody" box for the
usual case of noticing a new face in the photo already on screen. Above it,
only while something is in it, the `!dodaj` queue: one card per photo with
the sender's people and labels pre-ticked, **Approve** or **Reject** — the
only door into the album from the chat (`DOCS-add-photo.md`).
`--untagged` is a *view filter*, never a shorter list — `/api/album/media` and
the tag route both index the full album, so filtering server-side would show
item *i* of the album while tagging item *i* of the filtered set. Anything
seen this session stays navigable even once it has tags, or tagging the photo
you are looking at would remove it from the list you are walking.
## Analytics
The first tab: what the group actually asks the bot for. Read-only, and the
only thing on this page that is worth nothing to an attacker and everything
to a maintainer -- so there is deliberately no write route under it.
`?group=` is optional on every route. Absent means the all-groups roll-up,
and it runs the *same SQL* rather than a second query that would drift.
Every panel carries the window it covers and the three source counts, because
two of the three classes of row were reconstructed rather than observed.
`DOCS-analytics.md` says what each class knows and what it does not.
Charts are Nivo, always the fixed-size components: the `Responsive*` ones
measure their container, and a jsdom container has none, so the whole test
suite would assert against empty divs.
## Commands
The second tab is the switchboard: one row per command the bot is serving
right now, each with an on/off switch, an optional "off until" moment and a
cooldown that overrides the bot's default for that one command. Every row has
its own Save, because a switch is thrown for one command at a time.
No switch goes through `/_reload`. The bot keeps them in its own SQLite
(`command_settings`) and reads the row on every attempt, so a switch is live
on the next message -- `DOCS-reload.md` is about the *table*, and a switch
does not touch it. A command that is off answers `BEEP BOOP !kot WYŁĄCZONE`
(with `DO HH:MM` for a timed switch) rather than staying silent, so nobody
reports it broken; the refusal is filed under `refused_off`, counted by the
Analytics tab with the other refusals, and does not start the cooldown.
The list is the bot's, fetched live, which is why the page says so when the
bot is still connecting: there is no table to list until it has one. `!cisza`
and `!odcisz` have no cooldown to override, so the field is disabled with
"none" in it: a number typed there would be a lie the bot ignored.
The one thing in a row that *does* go through `/_reload` is the aliases
field: the operator's own comma-separated trigger words for that command.
They are data the table is built from, so `PUT /api/aliases/{name}` writes
`aliases_custom.json` in the bot's state directory and then asks the bot to
rebuild. Write and reload are one step on purpose: whenever the answer is
anything but `reloaded` -- a word another command has, a name no command
has, a bot still connecting -- the file is put back byte for byte and the
bot's own sentence is shown next to the field, so what is on disk is always
what the bot serves and a typo cannot poison every later reload. A missing
`!` is added, a word with a space is refused before any write, and an empty
field removes the command's custom words.
## Ideas
The fourth tab lists what the group has asked for: `!zaproponuj` notes with
the text people wrote, and every unrecognised `!word` with a count and the
messages it came in.
The **Export** button writes that to the state directory — `$STATE_DIRECTORY`,
else `$XDG_DATA_HOME/signal-bot/` — never into this repository. Two reasons,
both about a public git tree that has leaked real names once: a file the group
can write into would wedge `check_personal_data` the moment somebody's name
landed in it, refusing every commit until a human edited it by hand; and the
gate learns the names it refuses from the live database, so it catches `anna`
and cannot catch `!annna`, which is what an unrecognised-command corpus is full
of. So the export carries deliberate notes in full and unrecognised words as
counts only. Copy what is worth keeping into the tree yourself.
## Routes
Everything under `/api`. The page itself is served from `web/dist` at every
other path, so a reload on a sub-view works.
| Method | Path | What it does |
|---|---|---|
| GET | `/api/status` | Proxies the bot's `/healthz`. Unreachable is a state, never an error. |
| GET | `/api/groups` | The live listing, or `[]` when the container is down. |
| GET | `/api/config` | The resolved config. The number is cut to its last four digits. |
| GET / PUT | `/api/env` | Read and write `.env`, per the rules above. |
| POST | `/api/selftest` | Proxied to the bot verbatim, including its 403. |
| POST | `/api/restart-container` | `docker restart`, through `signal_bot/external.py`. |
| GET | `/api/album/state` | The album, the roster and the groups to pick from. |
| POST | `/api/album/tag` | Record who is in one file. |
| GET | `/api/album/summary` | What each command would answer with. |
| POST | `/api/album/setup` | Write the roster, then prove the bot could load it. |
| POST | `/api/album/person` | Add one person mid-session. |
| GET | `/api/album/media/{index}` | One item, **by position**. |
| GET | `/api/album/pending`, `…/pending/{name}/media` | The `!dodaj` queue, oldest first and naming no sender; one queued file, by the name the listing gave. |
| POST | `/api/album/pending/{name}/{approve,reject}` | Into the album with the ticked people and labels, then the bot reloads (a refused reload is reported, not rolled back); or deleted. |
| GET | `/api/suggestions` | Filed ideas and unrecognised-word tallies, bodies included. |
| GET | `/api/commands` | Proxies the bot's `/_commands`: the live table with each command's switches. |
| PUT | `/api/commands/{name}` | One command's switches, forwarded to the bot; its 404/503 come back unwrapped. |
| GET | `/api/aliases` | The operator's custom aliases, by command name; 409 if the file cannot be read. |
| PUT | `/api/aliases/{name}` | Writes the file and reloads the bot; anything but `reloaded` restores the file and passes the bot's reason on. |
| GET | `/api/analytics/groups` | Every group with a command history, for the picker. |
| GET | `/api/analytics/{overview,time,health,demand,behaviour,content}` | One per panel, all read-only, all taking an optional `?group=`. |
| POST | `/api/suggestions/export` | Write those to a file **outside the repository**. |
That last one is not a style choice. A filename supplied by a client is a
path, and this directory holds private photographs; the route takes an `int`,
so FastAPI refuses a name with **422** before any handler code runs.
## Building it
`dist/` is a build output and this repository does not commit those.
`scripts/install_service.sh` builds it, and verifies the artefact rather than
trusting the exit code. Until it has been built the page answers **503** with
the command to run, because `StaticFiles` raises at construction on a missing
directory — mounting it unconditionally would take the app down on a fresh
clone, and in CI, where pytest never runs `pnpm build`.
```sh
pnpm --dir web install --frozen-lockfile
pnpm --dir web build
pnpm --dir web lint # tsc --noEmit + eslint strictTypeChecked
pnpm --dir web coverage # vitest, 100% with no exclude list
```
React 19, Vite 8, TypeScript **6.0.3**, ESLint 10 with `strictTypeChecked` +
`stylisticTypeChecked` + unicorn + sonarjs + perfectionist, Vitest 4 at 100%
on all four metrics. TypeScript stays on 6.0.3 because
`typescript-eslint@8.68.0` declares `typescript: ">=4.8.4 <6.1.0"`; the pin is
fleet-wide in `~/utils/dependency-freshness.allowlist.yaml` and clears itself
when the linter catches up. The compiler waits for the linter.
`web/tsconfig.json` is strict JSON, not JSONC: the `check-json` hook is
generic and a commentable tsconfig is a convention, not a requirement. What
the comments used to say: `lib` is `ESNext` while `target` stays `ES2023`, so
iterator helpers are declared without being emitted for; and every strictness
flag is on, `noUncheckedIndexedAccess` and `exactOptionalPropertyTypes` included.
There is **no coverage exclude list**, including for `src/main.tsx`. A
coverage omit is a suppression, and "the DOM bootstrap is awkward to assert"
is the same sentence as "this code is untested" — so the bootstrap is a shim
thin enough to test in jsdom instead.

62
README.md
View File

@ -1,7 +1,6 @@
# signal-bot
A Signal group bot: `!cat` / `!dog` / `!gekon` / `!pułapka` / `!sztuka` /
`!wiersz` / `!papież`, and group statistics. Data stays out: `DOCS-privacy.md`.
A Signal group bot: `!cat` / `!dog` / `!gekon` image commands and a daily tally.
Runs on its **own registered Signal number** (or, if you prefer, as a linked
device on an existing account) via
@ -56,7 +55,7 @@ expect:
- { kind: image, to: reply }
```
It enforces a 40-second budget on itself. Give each scenario its own `uuid`: one bot process serves them all and the
Give each scenario its own `uuid`: one bot process serves them all and the
cooldown lives in SQLite, so a shared actor leaks state between files.
## Run it for real
@ -170,12 +169,18 @@ its own number and no access to your chats at all.
### Why not Shelter / a second Signal app on the phone
Two reasons, both verified rather than assumed. **A second app instance is not
a second account** — every Signal account needs its own phone number, so
Shelter never removed the actual blocker. And **it cannot be installed anyway**
on a phone with a Device Owner: Android then enables `no_add_managed_profile`
and its two siblings on every user, killing Shelter, Island, Private Space and
cloning alike. Unchanged through Android 16.
Two independent reasons, both verified rather than assumed:
1. **A second app instance is not a second account.** Every Signal account
needs its own phone number, so Shelter never removed the actual blocker.
Signal's phone-number-free "Signal Login" is announced but not shipped.
2. **It cannot be installed anyway** on a phone with a Device Owner. Setting a
Device Owner makes Android enable `no_add_managed_profile`,
`no_add_private_profile` and `no_add_clone_profile` on every user — killing
Shelter, Island, Private Space and app cloning alike. Verified on-device:
`pm create-user --profileOf 0 --managed` returns
`Cannot add user: no_add_managed_profile is enabled`. This is the Android 11
COMP deprecation and is unchanged through Android 16.
## When it cannot hear, and who may add it
@ -187,12 +192,12 @@ container, and `TRUSTED_INVITERS` decides who may add this bot to a group:
## Commands
`!pomoc` (or `!help`) prints every command **in the group**, in Polish, one
line of description each. It is built from the registry at run time and
`CommandRegistry.register` takes that description as a required argument, so a
command cannot be added without appearing there. Every trigger that resolves,
what the bot now answers to a `!word` nothing claims, and why the printed
spelling is not one of them: [DOCS-commands.md](DOCS-commands.md).
| Command | Aliases |
|---|---|
| cat | `!kot` `!koty` `!kots` `!cat` `!cats` `!meow` `!miau` + styled variants |
| dog | `!pies` `!psy` `!dog` `!dogs` `!woof` `!szczek` + styled variants |
| gecko | `!gekon` `!gekony` `!geko` `!gecko` `!geckon` `!geckos` |
| pope | `!papież` `!papiez` `!papa` `!pope` `!popes` `!pontifex` + styled variants |
Aliases live in `signal_bot/aliases.json`, not in source. Each command is rate-limited per
user, per command — the original bot shared one 10-second window across everybody.
@ -216,25 +221,20 @@ quietly posting the day's report into one group's chat.
The `per_group_isolation` scenario is the gate: one person asks in both groups
inside a single cooldown window and must get two images, one in each.
## Statistics
## Daily tally
One row of metadata per message — sender, Warsaw day, hour, media, length —
and **never the message body**. At `REPORT_HOUR:REPORT_MINUTE` (default 21:37)
each group gets its report for the previous full calendar day: silent members
at 0, up to three highlights. Sundays close a week, month ends a month. Ask it
things: `!staty` `!wczoraj` `!tydzień` `!miesiąc` `!rok` `!rekordy` `!korona`
`!dni` `!ja`. Full behaviour: `DOCS-statistics.md`.
## Managing it
`./web.sh` serves one page on `127.0.0.1:8140`: health, the configuration (editable) and the
album's tagger. Started by hand and stopped again, because it can rewrite `.env`: [DOCS-web-ui.md](DOCS-web-ui.md).
A tagged person goes live with `scripts/reload.sh`, no restart; code with `--code`: [DOCS-reload.md](DOCS-reload.md).
Every non-sticker, non-reaction message in a watched group increments its
sender's count **for that group**. At `REPORT_HOUR:REPORT_MINUTE` (default
21:37) each group is posted its own tally, in its own chat, and reset. A group
where nobody spoke is logged and **not** posted into — otherwise every quiet
group the bot has ever been added to would get a nightly ping. A group that
refuses the report keeps its counts and does not cost the others theirs.
Counts live in SQLite and survive a restart.
## Develop
```sh
pytest # 100% branch coverage, no network
pytest # 377 tests, 100% branch coverage, no network
pre-commit run --all-files # ruff ALL + mypy strict + 250-line cap + gates
./scripts/e2e.sh # the real bot against the fake server
```
@ -242,8 +242,8 @@ pre-commit run --all-files # ruff ALL + mypy strict + 250-line cap + gates
No `# noqa`, no `# type: ignore`, no coverage omits. Never bump `SCHEMA_VERSION` to add a table —
read `signal_bot/schema.py` first; a bump DROPs every table and throws the live tallies away, while
`CREATE TABLE IF NOT EXISTS` is additive and safe. Wire format: [DOCS-signal-envelopes.md](DOCS-signal-envelopes.md).
Releases are named after the popes, in order and in Latin — 0.1.4 is «Eleutherius» — and a bump publishes to the group as the pope's portrait, name and article: [DOCS-changelog.md](DOCS-changelog.md). Latency: [DOCS-latency.md](DOCS-latency.md).
Signal never delivers an account its own sends: `scripts/selftest.sh testbot '!papież' 3` drives the live dispatcher instead, [DOCS-selftest.md](DOCS-selftest.md). The page has its own gates: `pnpm --dir web lint` and `pnpm --dir web coverage`, both at 100% with no excludes.
Releases are named after the popes, in order, and a bump publishes to the group: [DOCS-changelog.md](DOCS-changelog.md). Latency: [DOCS-latency.md](DOCS-latency.md).
Signal never delivers an account its own sends: `scripts/selftest.sh testbot '!papież' 3` drives the live dispatcher instead, [DOCS-selftest.md](DOCS-selftest.md).
## Licence

BIN
bottles/1/7e946f8d-f1a3-40f9-8d1e-b9f4139ab051/indexeddb.sqlite Normal file
View File

Binary file not shown.

BIN
bottles/1/7e946f8d-f1a3-40f9-8d1e-b9f4139ab051/indexeddb.sqlite-shm Normal file
View File

Binary file not shown.

BIN
bottles/1/7e946f8d-f1a3-40f9-8d1e-b9f4139ab051/indexeddb.sqlite-wal Normal file
View File

Binary file not shown.

9
fake_signal/approved/dodaj_again_is_a_duplicate.yaml
View File

@ -1,9 +0,0 @@
# Content-addressed: the same bytes sent a second time, after approval, are
# recognised as already in the album -- the queue is not a way to store a
# photo twice. tool/e2e_media.py re-seeds the fake's store first, because
# the bot deleted the container's copy the first time.
name: "the same photo sent again after approval is already in the album"
inject:
- { text: "!dodaj wakacje", uuid: uuid-approved-again, name: ApprovedAgain, attachment: att-custom }
expect:
- { kind: text, to: reply, contains: "JUŻ JEST W ALBUMIE" }

9
fake_signal/approved/random_finds_the_approved_label.yaml
View File

@ -1,9 +0,0 @@
# The second half of scenarios/pack_add_custom.yaml. Replayed only after
# scripts/e2e.sh has approved the queue through the management page and the
# bot has rebuilt its table: the word that was NIE MAM TAKIEGO ZDJĘCIA a
# moment ago now opens a photo, with no restart in between.
name: "!losowe finds a label the moment it is approved"
inject:
- { text: "!losowe wakacje", uuid: uuid-approved-random, name: ApprovedRandom }
expect:
- { kind: image, to: reply }

87
fake_signal/cli.py
View File

@ -10,41 +10,13 @@ from __future__ import annotations
import argparse
import asyncio
import sys
import time
from pathlib import Path
from typing import Any
import httpx
from fake_signal.scenario import Scenario, check, load_scenario
# Measured, not guessed. A surplus send is *always* detected whatever this is,
# because collect() re-reads after the expected count arrives -- injecting a
# deliberate double-send proved that at every value tried, six runs, all red.
# What this buys is **attribution**: whether the surplus is blamed on the
# scenario that caused it or on the innocent one that follows. Measured with a
# surplus delayed by hand:
#
# surplus at +0.05s / +0.25s blamed correctly at 0.35 and at 0.15
# surplus at +0.50s blamed correctly at 0.35, wrongly at 0.15
#
# 0.15 halves the replay (35s -> 19s, against a 40s budget) and only
# misattributes a surplus arriving half a second late, which took an explicit
# asyncio.sleep to construct -- a real double-send comes from the same handler
# microseconds apart. Raise it with --settle if a slow machine starts blaming
# the wrong scenario; that symptom is what this number is for.
SETTLE_SECONDS = 0.15
# How long a scenario may wait for the sends it expects before giving up.
#
# A single fixed settle cannot cover both a reply that is a dictionary lookup
# and one that fetches an image and shells out to exiftool, and the failure
# that produced was not "slow" but "wrong": the late send landed after the
# NEXT scenario's reset and was counted against that one instead. So two
# different scenarios failed per run, never the same two twice -- one with
# "captured 0" and one with "captured 2". Waiting for the count is a
# guarantee; a bigger number is a guess that fails again on a slower machine.
PATIENCE_SECONDS = 15.0
POLL_SECONDS = 0.05
SETTLE_SECONDS = 0.35
def _emit(line: str) -> None:
@ -79,51 +51,8 @@ def resolve_groups(args: argparse.Namespace) -> dict[str, str]:
return groups
async def captured(client: httpx.AsyncClient) -> list[dict[str, Any]]:
"""Every send the fake server has recorded since the last reset."""
response = await client.get("/_control/sent")
response.raise_for_status()
sends: list[dict[str, Any]] = response.json()
return sends
async def collect(
client: httpx.AsyncClient, want: int, *, settle: float, patience: float
) -> list[dict[str, Any]]:
"""Wait for the sends a scenario expects, then for a beat of quiet.
The quiet beat is not politeness: returning the moment the expected count
arrives would make "expected 1, captured 2" unobservable, because the
surplus send is exactly the one that has not landed yet. A scenario that
expects nothing skips the poll entirely and waits out the settle, which
is the only way to assert that nothing happens.
Args:
client: A client pointed at the fake server's control plane.
want: How many sends the scenario expects.
settle: How long "quiet" is.
patience: How long to wait for ``want`` sends before giving up.
Returns:
Every send recorded, which may be more or fewer than expected -- the
comparison is somebody else's job.
"""
deadline = time.monotonic() + patience
sends = await captured(client)
while len(sends) < want and time.monotonic() < deadline:
await asyncio.sleep(POLL_SECONDS)
sends = await captured(client)
await asyncio.sleep(settle)
return await captured(client)
async def run_scenario(
base_url: str,
scenario: Scenario,
groups: dict[str, str],
*,
settle: float,
patience: float = PATIENCE_SECONDS,
base_url: str, scenario: Scenario, groups: dict[str, str], *, settle: float
) -> list[str]:
"""Inject every frame, wait for the bot to react, then compare."""
async with httpx.AsyncClient(base_url=base_url, timeout=10.0) as client:
@ -138,8 +67,9 @@ async def run_scenario(
for injection in scenario.inject:
await client.post("/_control/inject", json={"raw": injection.to_frame(groups)})
await asyncio.sleep(settle)
sent = await collect(client, len(scenario.expect), settle=settle, patience=patience)
return check(scenario, sent, groups)
response = await client.get("/_control/sent")
response.raise_for_status()
return check(scenario, response.json(), groups)
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
@ -149,9 +79,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser.add_argument("--base-url", default="http://127.0.0.1:8099")
parser.add_argument("--group", action="append", default=[], metavar="NAME=ID")
parser.add_argument("--settle", type=float, default=SETTLE_SECONDS)
# Overridable so the tests can drive the give-up path without waiting it
# out, and so a slow machine can be given longer without editing source.
parser.add_argument("--patience", type=float, default=PATIENCE_SECONDS)
return parser.parse_args(argv)
@ -161,9 +88,7 @@ async def _main_async(args: argparse.Namespace) -> int:
failed = 0
for path in args.scenarios:
scenario = load_scenario(path)
failures = await run_scenario(
args.base_url, scenario, groups, settle=args.settle, patience=args.patience
)
failures = await run_scenario(args.base_url, scenario, groups, settle=args.settle)
if failures:
failed += 1
_emit(f"FAIL {scenario.name}")

7
fake_signal/proofs/alias_after.yaml
View File

@ -1,7 +0,0 @@
# After the reload, the operator's spelling is !kot: the same process that
# just said "Nie znam" posts a cat.
name: "!e2ekot posts a cat once the alias file is live"
inject:
- { text: "!e2ekot", uuid: uuid-alias-after, name: AliasAfter }
expect:
- { kind: image, to: reply }

8
fake_signal/proofs/alias_before.yaml
View File

@ -1,8 +0,0 @@
# Not in scenarios/: the pair brackets the alias file that
# scripts/e2e_proofs.sh writes and reloads in between. Before it, !e2ekot
# is a word nobody registered.
name: "!e2ekot is unknown before the alias file exists"
inject:
- { text: "!e2ekot", uuid: uuid-alias, name: Alias }
expect:
- { kind: text, to: reply, contains: "Nie znam !e2ekot" }

8
fake_signal/proofs/alias_kept.yaml
View File

@ -1,8 +0,0 @@
# Replayed after a bad alias file is refused: the table that answered
# alias_after.yaml is the one still serving. Its own actor, because the
# cooldown lives in SQLite and alias_after's uuid is inside its window.
name: "!e2ekot still posts a cat while a bad alias file is refused"
inject:
- { text: "!e2ekot", uuid: uuid-alias-kept, name: AliasKept }
expect:
- { kind: image, to: reply }

7
fake_signal/proofs/reload_after.yaml
View File

@ -1,7 +0,0 @@
# The other half of the bracket: the same process, one POST /_reload later,
# and the person tagged into the album meanwhile answers with a photo.
name: "!delta posts a photo after the reload"
inject:
- { text: "!delta", uuid: uuid-reload-after, name: ReloadAfter }
expect:
- { kind: image, to: reply }

8
fake_signal/proofs/reload_before.yaml
View File

@ -1,8 +0,0 @@
# Not in scenarios/: that glob replays against the pack as first written,
# and this pair brackets the POST /_reload that scripts/e2e_proofs.sh makes
# in between. Before it, !delta is a word the bot has never heard.
name: "!delta is unknown before the reload"
inject:
- { text: "!delta", uuid: uuid-reload-before, name: ReloadBefore }
expect:
- { kind: text, to: reply, contains: "Nie znam !delta" }

9
fake_signal/proofs/switch_off.yaml
View File

@ -1,9 +0,0 @@
# Not in scenarios/: this pair brackets the PUT /_commands/cat that
# scripts/e2e_proofs.sh makes in between, with no reload and no restart.
# After the switch is thrown, the same process that has been posting cats
# all run refuses in so many words.
name: "!cat switched off answers WYŁĄCZONE instead of a photo"
inject:
- { text: "!cat", uuid: uuid-switch, name: Switch }
expect:
- { kind: text, to: reply, contains: "!kot WYŁĄCZONE" }

9
fake_signal/proofs/switch_on.yaml
View File

@ -1,9 +0,0 @@
# The second half of the pair: the switch is back on, and the *same* actor
# gets the photo. Same uuid on purpose -- a refusal must not start the
# cooldown, or switching a command back on would leave everyone who tried
# it meanwhile waiting out a window they never used.
name: "!cat switched back on posts a photo to the actor it just refused"
inject:
- { text: "!cat", uuid: uuid-switch, name: Switch }
expect:
- { kind: image, to: reply }

47
fake_signal/routes_api.py
View File

@ -3,16 +3,13 @@
from __future__ import annotations
from typing import Any, Final
from typing import Any
from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
from fastapi.responses import JSONResponse, Response
from fake_signal.state import PIXEL_PNG, State
# Read off the real container's /swagger/doc.json, not remembered.
REACTION_FIELDS: Final = frozenset({"reaction", "recipient", "target_author", "timestamp"})
def _state(request: Request) -> State:
"""Return the State attached to the running app."""
@ -60,49 +57,21 @@ def build_api_router() -> APIRouter:
_state(request).record_send(payload)
return JSONResponse({"timestamp": 1719177917717}, status_code=201)
@router.post("/v1/reactions/{number}")
async def react(request: Request, number: str, payload: dict[str, Any]) -> Response:
"""The container's api.SendReactionRequest: all four fields required."""
if REACTION_FIELDS - payload.keys():
return JSONResponse({"error": "missing reaction fields"}, status_code=400)
_state(request).record_reaction(number, payload)
return Response(status_code=204)
_add_attachment_routes(router)
_add_receive_route(router)
return router
def _add_attachment_routes(router: APIRouter) -> None:
"""Attach the attachment store's three endpoints.
Split out so the router builder stays under the complexity cap, exactly
as _add_receive_route already is.
"""
@router.get("/v1/attachments")
async def list_attachments(request: Request) -> list[str]:
return list(_state(request).attachments)
@router.get("/v1/attachments/{attachment_id}")
async def fetch_attachment(request: Request, attachment_id: str) -> Response:
"""Hand over one stored attachment, the way the container does."""
payload = _state(request).stored.get(attachment_id)
if payload is None:
return JSONResponse({"error": "no such attachment"}, status_code=404)
return Response(content=payload, media_type="application/octet-stream")
@router.delete("/v1/attachments/{attachment_id}")
async def delete_attachment(request: Request, attachment_id: str) -> Response:
state = _state(request)
if attachment_id in state.attachments:
state.attachments.remove(attachment_id)
# The bytes go too, which is what makes "the second copy is gone"
# something a scenario can assert rather than assume.
state.stored.pop(attachment_id, None)
attachments = _state(request).attachments
if attachment_id in attachments:
attachments.remove(attachment_id)
return Response(status_code=204)
_add_receive_route(router)
return router
def _add_receive_route(router: APIRouter) -> None:
"""Attach the WebSocket receive endpoint.

29
fake_signal/routes_control.py
View File

@ -3,7 +3,6 @@
from __future__ import annotations
import base64
from typing import TYPE_CHECKING, Any
from fastapi import APIRouter, Request
@ -80,32 +79,4 @@ def build_control_router() -> APIRouter:
async def ready(request: Request) -> dict[str, int]:
return {"sockets": len(_state(request).sockets)}
_add_attachment_routes(router)
return router
def _add_attachment_routes(router: APIRouter) -> None:
"""Attach the inbound-attachment control endpoints.
Split out to keep the builder under the complexity cap.
"""
@router.post("/attachment")
async def put_attachment(request: Request, body: dict[str, Any]) -> dict[str, int]:
"""Put one inbound attachment into the container's store.
base64 rather than raw bytes so a scenario's fixture can be generated
at run time by scripts/e2e.sh -- no binary file enters this repo.
"""
state = _state(request)
attachment_id = str(body["id"])
state.stored[attachment_id] = base64.b64decode(str(body["base64"]))
if attachment_id not in state.attachments:
state.attachments.append(attachment_id)
return {"stored": len(state.stored)}
@router.get("/attachments")
async def stored_attachments(request: Request) -> list[str]:
"""Which attachment ids still have bytes behind them."""
return sorted(_state(request).stored)

17
fake_signal/routes_fixtures.py
View File

@ -44,23 +44,6 @@ def build_fixture_router() -> APIRouter:
}
return {"query": {"pages": [page]}}
# Wolne Lektury serves the poem as plain text in a fixed shape: author,
# blank, title, the poem, then a footer behind a lone rule. The slug is
# echoed into the title so a scenario proves the bot asked for the poem
# it drew rather than a hard-coded one. The public-domain sentence is
# here because poem_command refuses anything without it.
@router.get("/poem-text")
async def poem_text(request: Request) -> Response:
slug = request.query_params.get("slug", "")
body = (
f"Adam Testowy\n\n{slug}\n\n\n"
"Pierwsza linijka wiersza,\n"
"druga linijka wiersza.\n\n"
"-----\n"
"Ten utwór jest w domenie publicznej.\n"
)
return Response(content=body, media_type="text/plain; charset=utf-8")
@router.get("/image.png")
async def image() -> Response:
return Response(content=PIXEL_PNG, media_type="image/png")

40
fake_signal/scenario.py
View File

@ -5,7 +5,7 @@ from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any
import yaml
@ -28,12 +28,6 @@ class Injection:
sticker: bool = False
reaction: bool = False
sync: bool = False
# The id of an attachment the scenario put in the store beforehand, via
# POST /_control/attachment. Named rather than inlined: the bytes are
# generated at run time by scripts/e2e.sh, because a fixture photo in
# this repository is exactly what the binary gate exists to prevent.
attachment: str | None = None
attachment_type: str = "image/jpeg"
def to_frame(self, groups: dict[str, str]) -> str:
"""Render this injection as a raw signal-cli envelope."""
@ -45,11 +39,6 @@ class Injection:
msg = f"unknown group alias {self.group!r}; known: {sorted(groups)}"
raise ScenarioError(msg)
payload["groupInfo"] = {"groupId": groups[self.group], "type": "DELIVER"}
if self.attachment is not None:
# Top level, never inside "quote": that is where a *reply* carries
# the quoted message's media, and !add_photo must never take a
# photo somebody else posted.
payload["attachments"] = [{"contentType": self.attachment_type, "id": self.attachment}]
if self.sticker:
payload["sticker"] = {"packId": "abc", "stickerId": 6}
if self.reaction:
@ -160,35 +149,16 @@ def load_scenario(path: Path) -> Scenario:
)
# How signal-cli-rest-api spells an attachment whose type the bot stated.
# Anything else is bare base64, which is what an ordinary photo travels as.
_VIDEO_PREFIX: Final = "data:video/"
def describe(payload: dict[str, Any]) -> tuple[str, str, str]:
"""Reduce a captured send to (kind, recipient, body) for comparison.
"video" is its own kind so a scenario can assert that a clip went out as
a clip: told it is a photo, Signal renders an unplayable file, and every
other check in this harness would still pass.
"""
if "reaction" in payload:
return "reaction", str(payload.get("recipient", "")), str(payload["reaction"])
attachments = payload.get("base64_attachments") or []
kind = "text"
if attachments:
kind = "video" if str(attachments[0]).startswith(_VIDEO_PREFIX) else "image"
"""Reduce a captured send to (kind, recipient, body) for comparison."""
kind = "image" if payload.get("base64_attachments") else "text"
recipients = payload.get("recipients") or [""]
return kind, str(recipients[0]), str(payload.get("message", ""))
def quotes(payload: dict[str, Any]) -> bool:
"""True when a captured send points at a specific message.
A reply points with ``quote_timestamp``; a reaction has nothing else to
do, and names its target as ``timestamp``.
"""
return bool(payload.get("quote_timestamp") or payload.get("timestamp"))
"""True when a captured send points at a specific message."""
return bool(payload.get("quote_timestamp"))
def check(scenario: Scenario, sent: list[dict[str, Any]], groups: dict[str, str]) -> list[str]:

17
fake_signal/scenarios/analytics_command.yaml
View File

@ -1,17 +0,0 @@
# Fresh uuid: one bot process serves every scenario and the cooldown lives in
# SQLite, so a shared actor would leak a refusal into this file.
#
# Two frames on purpose. The first is a command that leaves a row behind; the
# second asks about it. That is what makes this an end-to-end proof of the
# recording rather than of the rendering: a broken recorder answers the
# second frame with "nobody has used anything yet".
name: "!analityka answers with the history the frame before it just made"
inject:
- { text: "!gekon", uuid: uuid-nadia, name: Nadia }
- { text: "!analityka", uuid: uuid-nadia, name: Nadia }
# One expectation per send. "!gecko" rather than the heading, because the
# heading proves only that the command rendered -- naming the command from
# the frame before it proves the dispatcher actually recorded that frame.
expect:
- { kind: image, to: reply }
- { kind: text, to: reply, contains: "!gecko" }

5
fake_signal/scenarios/art_command.yaml
View File

@ -1,5 +0,0 @@
name: "!sztuka posts a work of art, captioned with what it is"
inject:
- { text: "!sztuka", uuid: uuid-gallery, name: Gallery }
expect:
- { kind: image, to: reply }

9
fake_signal/scenarios/chatter_is_silent.yaml
View File

@ -1,9 +0,0 @@
# Ordinary conversation is not a command attempt and never was. Kept as its
# own scenario when unknown_command.yaml stopped expecting silence, because
# this half of the old assertion is the half still worth pinning.
name: "ordinary chatter produces nothing"
inject:
- { text: "just chatting", uuid: uuid-alice, name: Alice }
- { text: "hej! super", uuid: uuid-alice, name: Alice }
- { text: "!!", uuid: uuid-alice, name: Alice }
expect: []

8
fake_signal/scenarios/did_you_mean.yaml
View File

@ -1,8 +0,0 @@
# The terminal's trick. "!gekno" is one transposition from "!gekon", which is
# why the distance is optimal string alignment rather than difflib's ratio --
# a swap costs one edit there and shreds the matching blocks difflib counts.
name: "a near miss is corrected"
inject:
- { text: "!gekno", uuid: uuid-alice, name: Alice }
expect:
- { kind: text, to: reply, contains: "!gekon" }

7
fake_signal/scenarios/food_today.yaml
View File

@ -1,7 +0,0 @@
# The operator's food log for today, read from the file the run wrote
# rather than the real one: the meal is invented, the day is real.
name: "!co_jadł lists what the food log holds for today"
inject:
- { text: "!co_jadł", uuid: uuid-alice, name: Alice }
expect:
- { kind: text, to: reply, contains: "carbonara — 350 g, 550 kcal (manual)" }

5
fake_signal/scenarios/good_bot.yaml
View File

@ -1,5 +0,0 @@
name: "a compliment gets an emoji on the compliment itself, not a message in the group"
inject:
- { text: "!good bot", uuid: uuid-fan, name: Fan }
expect:
- { kind: reaction, to: reply, quotes: true }

7
fake_signal/scenarios/help_command.yaml
View File

@ -1,7 +0,0 @@
# Fresh uuid: one bot process serves every scenario and the cooldown lives in
# SQLite, so a shared actor would leak a refusal into this file.
name: "!pomoc lists every command, including ones from other modules"
inject:
- { text: "!pomoc", uuid: uuid-helen, name: Helen }
expect:
- { kind: text, to: reply, contains: "!gekon" }

16
fake_signal/scenarios/mute_command.yaml
View File

@ -1,16 +0,0 @@
# One confirmation, then nothing -- not even for a command that would
# otherwise be served -- until the unmute, which is the only word that gets
# through. The suite runs TRUSTED_INVITERS empty, so anybody may end it,
# and the scenario MUST end unmuted: every later file shares this process
# and this group, and a mute left standing would silence all of them.
name: "!cisza silences the group until !odcisz"
inject:
- { text: "!cisza 1", uuid: uuid-mona, name: Mona }
- { text: "!cat", uuid: uuid-mona, name: Mona }
- { text: "!cta", uuid: uuid-mona, name: Mona }
- { text: "!odcisz", uuid: uuid-mona, name: Mona }
- { text: "!cat", uuid: uuid-mona, name: Mona }
expect:
- { kind: text, to: reply, contains: "CISZA PRZEZ 1 MINUTĘ" }
- { kind: text, to: reply, contains: "WRACAM" }
- { kind: image, to: reply }

12
fake_signal/scenarios/pack_add_custom.yaml
View File

@ -1,12 +0,0 @@
# A label matching nobody is a proposal, not a tag: the photo waits in the
# queue and !losowe cannot find the word until somebody approves it on the
# management page. fake_signal/approved/ replays the other half of this,
# after scripts/e2e.sh has approved the queue. The typed label must appear
# in neither reply.
name: "!dodaj with a new label queues the photo, and !losowe cannot see it yet"
inject:
- { text: "!dodaj wakacje", uuid: uuid-add-custom, name: AddCustom, attachment: att-custom }
- { text: "!losowe wakacje", uuid: uuid-add-custom-b, name: AddCustomB }
expect:
- { kind: text, to: reply, contains: "czeka na zatwierdzenie: 1" }
- { kind: text, to: reply, contains: "NIE MAM TAKIEGO ZDJĘCIA" }

14
fake_signal/scenarios/pack_add_heic.yaml
View File

@ -1,14 +0,0 @@
# The iPhone default. Its HEVC parameter set survives `-all=` and is on the
# structural allowlist by kuhy's decision of 2026-08-31; the GPS and the Make
# do not survive, which is what e2e.sh re-reads the queued file to prove.
name: "a geotagged heic is accepted and waits without its GPS"
inject:
- {
text: "!dodaj gamma", uuid: uuid-add-heic, name: AddHeic,
attachment: att-heic, attachment_type: "image/heic",
}
# Tagged to gamma rather than alfa on purpose: pack_command.yaml asserts
# !alfa returns an image, and the pick is random -- putting a clip in p1
# would make that scenario fail about half the time once approved.
expect:
- { kind: text, to: reply, contains: "czeka na zatwierdzenie" }

9
fake_signal/scenarios/pack_add_no_attachment.yaml
View File

@ -1,9 +0,0 @@
# The proposed label is held in memory and reaches neither custom.json nor
# the queue: this is the case that must leave the album byte-identical.
# scripts/e2e.sh checks the files themselves, because a reply cannot prove
# what was not written.
name: "!dodaj with no attachment changes nothing on disk"
inject:
- { text: "!dodaj piesek", uuid: uuid-add-none, name: AddNone }
expect:
- { kind: text, to: reply, contains: "DOŁĄCZ ZDJĘCIE" }

10
fake_signal/scenarios/pack_add_person.yaml
View File

@ -1,10 +0,0 @@
# The feature in the shape the group will actually use it: a photo captioned
# with somebody's name. It is queued, not stored -- the attachment carries a
# real APP1 Exif segment (tool/e2e_media.py puts GPS, a Make and an XMP
# creator in it), and scripts/e2e.sh re-reads the queued file to prove the
# strip happened before the file waited anywhere. The reply is a count.
name: "!dodaj with an existing person queues the photo for approval"
inject:
- { text: "!dodaj alfa", uuid: uuid-add-person, name: AddPerson, attachment: att-person }
expect:
- { kind: text, to: reply, contains: "czeka na zatwierdzenie: 1" }

10
fake_signal/scenarios/pack_add_rejected.yaml
View File

@ -1,10 +0,0 @@
# Sniffed on the bytes, never on the filename or the stated contentType --
# this one claims to be a photo and is a text file.
name: "a file that is not media is refused and never queued"
inject:
- {
text: "!dodaj alfa", uuid: uuid-add-junk, name: AddJunk,
attachment: att-junk, attachment_type: "image/jpeg",
}
expect:
- { kind: text, to: reply, contains: "NIE PRZYJMUJĘ" }

10
fake_signal/scenarios/pack_add_scoping.yaml
View File

@ -1,10 +0,0 @@
# The privacy guarantee, for the write path this time. In a group the album
# was not given to, !dodaj is not a command at all: nothing is queued, nothing
# is answered, and nothing names the album in a group that does not have one.
name: "!dodaj does not exist in a group the album was not given to"
inject:
- {
text: "!dodaj alfa", group: second, uuid: uuid-add-scope, name: AddScope,
attachment: att-second,
}
expect: []

12
fake_signal/scenarios/pack_add_svg.yaml
View File

@ -1,12 +0,0 @@
# kuhy's decision of 2026-08-31: exiftool 13.55 can write neither .svg nor
# .webm, and both stay on the allowlist and are queued exactly as they
# arrive -- metadata included. This scenario exists so that hole is
# demonstrated by the suite rather than only described in the docs.
name: "an svg is accepted and queued unstripped, by decision"
inject:
- {
text: "!dodaj alfa", uuid: uuid-add-svg, name: AddSvg,
attachment: att-svg, attachment_type: "image/svg+xml",
}
expect:
- { kind: text, to: reply, contains: "czeka na zatwierdzenie" }

15
fake_signal/scenarios/pack_add_video.yaml
View File

@ -1,15 +0,0 @@
# An mp4 geotagged the way a phone does it: a QuickTime udta atom, which a
# naive "-GPS:all" gate passes straight through because exiftool files it
# under QuickTime and not under GPS. It is accepted, and scripts/e2e.sh then
# re-reads the queued file and fails the run if the atom survived.
name: "a geotagged mp4 is stripped of its QuickTime atom and queued"
inject:
- {
text: "!dodaj gamma", uuid: uuid-add-video, name: AddVideo,
attachment: att-video, attachment_type: "video/mp4",
}
# Tagged to gamma rather than alfa on purpose: pack_command.yaml asserts
# !alfa returns an image, and the pick is random -- putting a clip in p1
# would make that scenario fail about half the time once approved.
expect:
- { kind: text, to: reply, contains: "czeka na zatwierdzenie" }

7
fake_signal/scenarios/pack_combination.yaml
View File

@ -1,7 +0,0 @@
# The whole point of the feature: several names mean "a photo with all of
# them", and only both.png is tagged with p1 and p2 at once.
name: "two names ask for a photo with both people on it"
inject:
- { text: "!alfa beta", uuid: uuid-pack-two, name: Pack2 }
expect:
- { kind: image, to: reply }

7
fake_signal/scenarios/pack_command.yaml
View File

@ -1,7 +0,0 @@
# The private album, against a synthetic pack the e2e script writes: the real
# one lives outside this repository and must never be needed to run the suite.
name: "!alfa posts a photo from the private album"
inject:
- { text: "!alfa", uuid: uuid-pack-one, name: Pack1 }
expect:
- { kind: image, to: reply }

7
fake_signal/scenarios/pack_no_photo.yaml
View File

@ -1,7 +0,0 @@
# A combination nobody photographed is answered, not met with silence: from
# the group's side, silence is indistinguishable from a broken bot.
name: "a combination with no photo gets one line and no attachment"
inject:
- { text: "!beta gamma", uuid: uuid-pack-three, name: Pack3 }
expect:
- { kind: text, to: reply, contains: "NIE MAM TAKIEGO" }

7
fake_signal/scenarios/pack_scoping.yaml
View File

@ -1,7 +0,0 @@
# The privacy guarantee. In a group the pack does not list, "!alfa" is not a
# command at all: no photo, no refusal, nothing -- a refusal would itself
# announce that the album exists.
name: "the album does not exist in a group it was not given to"
inject:
- { text: "!alfa", group: second, uuid: uuid-pack-five, name: Pack5 }
expect: []

7
fake_signal/scenarios/pack_video.yaml
View File

@ -1,7 +0,0 @@
# Told it is a photo, Signal renders a clip as a file nobody can play. The
# harness reads the data URI the bot sent, so this fails if the type is lost.
name: "a tagged video leaves as a video, not as a photo"
inject:
- { text: "!wideo", uuid: uuid-pack-four, name: Pack4 }
expect:
- { kind: video, to: reply }

5
fake_signal/scenarios/poem_command.yaml
View File

@ -1,5 +0,0 @@
name: "!wiersz answers with the poem itself, no attachment and no network"
inject:
- { text: "!wiersz", uuid: uuid-poet, name: Poet }
expect:
- { kind: text, to: reply }

7
fake_signal/scenarios/suggest.yaml
View File

@ -1,7 +0,0 @@
# Filing an idea from inside the group. The word must not already be a
# command anywhere, so this one is invented rather than borrowed.
name: "!zaproponuj files an idea"
inject:
- { text: "!zaproponuj !pogoda pokaz pogode na jutro", uuid: uuid-alice, name: Alice }
expect:
- { kind: text, to: reply, contains: "Zapisane" }

5
fake_signal/scenarios/trap_command.yaml
View File

@ -1,5 +0,0 @@
name: "!pułapka posts a photograph of the device, captioned with what it is"
inject:
- { text: "!pułapka", uuid: uuid-trap, name: Trapper }
expect:
- { kind: image, to: reply }

9
fake_signal/scenarios/unknown_command.yaml
View File

@ -1,8 +1,5 @@
# A "!" word nothing claims. Until 0.1.x this was silent, which made a typo
# indistinguishable from the bot being down. Nothing is close enough to
# "!nosuchcommand" to suggest, so the answer points at !zaproponuj instead.
name: "an unrecognised command is answered, not ignored"
name: "chatter and unknown commands produce nothing"
inject:
- { text: "just chatting", uuid: uuid-alice, name: Alice }
- { text: "!nosuchcommand", uuid: uuid-alice, name: Alice }
expect:
- { kind: text, to: reply, contains: "!zaproponuj" }
expect: []

9
fake_signal/scenarios/unknown_command_scoping.yaml
View File

@ -1,9 +0,0 @@
# The privacy guarantee again, this time for the near-miss path. The album's
# triggers are real people's names. In a group without the album "!alfa" is
# unclaimed *here*, so a check scoped to this group would answer it and store
# it -- announcing the album and putting a name in the database. The check is
# global instead: claimed anywhere at all means silence.
name: "an album name is not echoed in a group without the album"
inject:
- { text: "!alfa", group: second, uuid: uuid-pack-five, name: Pack5 }
expect: []

6
fake_signal/scenarios/workout_today.yaml
View File

@ -1,6 +0,0 @@
# Today's workout with every field it carries, from the run's own log.
name: "!trening prints today's workout with all its fields"
inject:
- { text: "!trening", uuid: uuid-alice, name: Alice }
expect:
- { kind: text, to: reply, contains: "duration minutes: 112.7" }

15
fake_signal/state.py
View File

@ -24,12 +24,6 @@ class State:
sockets: list[WebSocket] = field(default_factory=list)
sent: list[dict[str, Any]] = field(default_factory=list)
attachments: list[str] = field(default_factory=list)
# Inbound attachment bytes by id, which the real container keeps in
# its own store. Without these the fake can name an attachment on an
# envelope but never hand it over, so !add_photo could not be proven
# offline at all. Deliberately not cleared by reset(): a scenario
# uploads its fixture before the run, not between injections.
stored: dict[str, bytes] = field(default_factory=dict)
mode: str = "json-rpc"
base_url: str = "http://127.0.0.1:8099"
# Registration and group state, so the registration runbook can be
@ -46,15 +40,6 @@ class State:
self.sent.clear()
self.attachments.clear()
def record_reaction(self, number: str, payload: dict[str, Any]) -> None:
"""Capture one reaction in the same list as the sends.
One list, because a scenario's ``expect`` is the bot's outbound
traffic in order, and "it reacted instead of replying" is exactly
the kind of claim that ordering has to be able to make.
"""
self.sent.append({"number": number, **payload})
def record_send(self, payload: dict[str, Any]) -> None:
"""Capture one outgoing message and mint ids for its attachments."""
self.sent.append(payload)

34
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "signal-bot"
version = "0.2.6"
version = "0.0.7"
description = "A Signal group bot: image commands and daily message tallies."
readme = "README.md"
license = "MIT"
@ -26,7 +26,7 @@ dev = [
"pytest==9.1.1",
"pytest-asyncio==1.4.0",
"pytest-cov==7.1.0",
"ruff==0.16.6",
"ruff==0.16.5",
"types-PyYAML==6.0.12.20260815",
]
@ -60,37 +60,7 @@ fixable = ["ALL"]
# Verified empirically, not copied: with this block removed, ruff reports
# exactly these four rules across tests/ and nothing else. SLF001 was in
# the original list and fired zero times, so it is gone.
[tool.ruff.lint.isort]
# venv_bootstrap sets up .venv as a side effect of being imported, so it has
# to be imported before anything that lives in .venv. Left to its own devices
# isort sorts it alphabetically -- after fastapi and httpx -- which silently
# breaks the thing it exists to do. Giving it its own section ahead of the
# standard library makes the formatter enforce the order instead of undoing
# it, which is better than a comment asking people not to reformat.
section-order = [
"future",
"bootstrap",
"standard-library",
"third-party",
"first-party",
"local-folder",
]
[tool.ruff.lint.isort.sections]
bootstrap = ["venv_bootstrap"]
[tool.ruff.lint.per-file-ignores]
# The single subprocess in this repository. S603 flags every subprocess call
# and has no code-level fix -- exiftool is a binary, and the pure-Python EXIF
# libraries handle neither HEIC nor MP4. signal_bot/external.py exists so this
# entry can name exactly one file: any other module that shells out fails the
# gate, which is the point. Do not widen this to a directory.
"signal_bot/external.py" = ["S603"]
# The bootstrap that installs .venv. It cannot route through external.py --
# that module lives inside the package it is installing -- so it is the one
# other place allowed to start a process. Kept to a single small file for
# exactly that reason. kuhy's call, 2026-08-31. Do not widen to a directory.
"tool/venv_bootstrap.py" = ["S603", "S606"]
"tests/**/*.py" = [
"ARG", # fixture and mock params exist for their side effect
"D", # test names are the documentation

26
scripts/check_no_binaries.sh
View File

@ -1,26 +0,0 @@
#!/bin/bash
# ============================================================================
# Thin delegate to the shared gate in ~/utils.
#
# Copying the logic here is what lets one repo's idea of the rule drift from
# every other repo's, so this script only locates the shared checker and
# forwards its arguments.
# ============================================================================
set -euo pipefail
readonly SHARED_GATE="${UTILS_ROOT:-$HOME/utils}/scripts/check_no_binaries.sh"
main() {
if [[ ! -x "$SHARED_GATE" ]]; then
echo "Error: shared gate not found at $SHARED_GATE" >&2
echo " Clone github.com/kuhyx/utils to ~/utils, or set" >&2
echo " UTILS_ROOT to where it lives." >&2
exit 1
fi
exec bash "$SHARED_GATE" "$@"
}
main "$@"

158
scripts/check_personal_data.py
View File

@ -1,158 +0,0 @@
#!/usr/bin/env python3
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Refuse to commit anything that identifies a real person.
This repository is public and has already leaked once: real first names, a
full name, two phone numbers and a group member's Signal uuid reached GitHub
and stayed there until the repository had to be deleted and recreated. Every
one of them arrived the same way -- somebody (a human once, this program's
author the second time) reached for a real name because it was the example
in front of them.
So it is a gate rather than a rule. Two halves, because they fail differently:
* **Names**, which cannot be listed here. The forbidden words are read at run
time from outside the repository -- the photo pack's manifest, the message
database, the local .env -- so the check knows them and the repository
never does. On a machine with none of those (CI, a fresh clone) this half
simply has nothing to compare against and says so.
* **Shapes**, which can. A phone number, a uuid and a Signal group id look
like themselves wherever they appear, so those are matched by pattern and
compared against the placeholders this repository is allowed to contain.
Usage:
scripts/check_personal_data.sh --all # every tracked file
scripts/check_personal_data.sh a.py b.md # named files
The shell wrapper is the entry point; it knows how to list what git tracks,
which keeps this half free of a subprocess call it would need a suppression
for.
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
from typing import Final
sys.path.insert(0, str(Path(__file__).resolve().parent))
from personal_data_sources import REPO, forbidden
ALLOWLIST: Final = REPO / ".personal-data-allowlist"
# Shapes that identify somebody wherever they turn up.
PATTERNS: Final[dict[str, re.Pattern[str]]] = {
"phone number": re.compile(r"\+\d{9,15}\b"),
"uuid": re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b"),
"signal group id": re.compile(r"\bgroup\.[A-Za-z0-9+/]{16,}={0,2}"),
}
SKIP_DIRS: Final = frozenset({".git", ".venv", "__pycache__", ".mypy_cache", ".ruff_cache"})
SKIP_SUFFIXES: Final = frozenset({".lock", ".bundle"})
def allowed(path: Path | None = None) -> set[str]:
"""Literals this repository may contain, lower-cased.
A plain line permits a value anywhere. A ``some/path::value`` line
permits it only in that file, which is how a name that is also a pope in
popes.json stays refused in every test fixture and document that has no
business carrying it.
"""
if not ALLOWLIST.exists():
return set()
permitted: set[str] = set()
where = str(path) if path is not None else ""
# The allowlist has to be able to name the values it permits, including
# the file-scoped ones, so inside it every listed value counts as
# permitted. It is the list of exceptions; it cannot be its own exception.
inside = path is not None and path.resolve() == ALLOWLIST
for raw in ALLOWLIST.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
scope, separator, value = line.partition("::")
if not separator:
permitted.add(line.lower())
elif inside or where.endswith(scope.strip()):
permitted.add(value.strip().lower())
return permitted
def scan(text: str, words: set[str], path: Path | None = None) -> list[tuple[int, str]]:
"""Return (line number, what matched) for every hit in one document."""
permitted = allowed(path)
words = {word for word in words if word not in permitted}
hits: list[tuple[int, str]] = []
for number, line in enumerate(text.splitlines(), start=1):
lowered = line.lower()
for word in words:
if re.search(rf"(?<![\w]){re.escape(word)}(?![\w])", lowered):
# The word itself is never printed: this output can end up in
# a CI log, and naming the name there would be the same leak
# by a longer route.
hits.append((number, "a name from your private data"))
break
hits.extend(
(number, f"{label} {match!r} is not in .personal-data-allowlist")
for label, pattern in PATTERNS.items()
for match in pattern.findall(line)
if match.lower() not in permitted
)
return hits
def readable(path: Path) -> str | None:
"""The file's text, or None when it is not text at all."""
if path.suffix in SKIP_SUFFIXES or any(part in SKIP_DIRS for part in path.parts):
return None
try:
return path.read_text(encoding="utf-8")
except OSError, UnicodeDecodeError:
return None
def main() -> int:
"""Check whatever was asked for, and explain any refusal."""
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("--files", nargs="*", type=Path, default=[])
parser.add_argument("--message", type=Path, default=None, help="a commit message file")
arguments = parser.parse_args()
words, sources = forbidden(allowed())
targets = list(arguments.files)
if arguments.message is not None:
targets.append(arguments.message)
failures = 0
for path in targets:
text = readable(path)
if text is None:
continue
for number, reason in scan(text, words, path):
sys.stdout.write(f"{path}:{number}: {reason}\n")
failures += 1
if failures:
sys.stdout.write(
f"\nRefused: {failures} line(s) carry personal data.\n"
"This repository is public. Use an invented name, a placeholder number, or\n"
"add the value to .personal-data-allowlist if it is genuinely not anybody's.\n"
)
return 1
if not words:
# Said out loud rather than passing quietly: on CI and on a fresh
# clone the name half of this check has nothing to compare against,
# and a gate that silently checks nothing is worse than no gate.
sys.stdout.write("no local private data to compare against; shapes only\n")
else:
sys.stdout.write(f"clean ({len(words)} name(s) from: {', '.join(sources)})\n")
return 0
if __name__ == "__main__":
raise SystemExit(main())

73
scripts/check_personal_data.sh
View File

@ -1,73 +0,0 @@
#!/bin/bash
# ============================================================================
# Refuse to commit anything that identifies a real person.
#
# The logic is check_personal_data.py; this half exists to list what git
# tracks, which keeps the Python free of a subprocess call it would otherwise
# need a lint suppression for.
#
# scripts/check_personal_data.sh --all every tracked file
# scripts/check_personal_data.sh a.py b.md named files
# scripts/check_personal_data.sh --message .git/COMMIT_EDITMSG
# ============================================================================
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly SCRIPT_DIR
REPO_ROOT="$(dirname "$SCRIPT_DIR")"
readonly REPO_ROOT
readonly CHECKER="${SCRIPT_DIR}/check_personal_data.py"
# The repo's own interpreter when there is one, so the check runs the same
# way under pre-commit, in CI and by hand.
PYTHON="${PYTHON:-${REPO_ROOT}/.venv/bin/python}"
if [[ ! -x "$PYTHON" ]]; then
PYTHON="$(command -v python3)"
fi
readonly PYTHON
usage() {
echo "Usage: $(basename "$0") [--all | --message FILE | FILE...]"
exit 0
}
main() {
if [[ $# -eq 0 ]]; then
usage
fi
case "$1" in
-h | --help)
usage
;;
--message)
if [[ $# -lt 2 ]]; then
echo "Error: --message needs a file" >&2
exit 1
fi
"$PYTHON" "$CHECKER" --message "$2"
;;
--all)
local files=()
mapfile -t files < <(git -C "$REPO_ROOT" ls-files)
if [[ ${#files[@]} -eq 0 ]]; then
echo "Error: git tracks no files here" >&2
exit 1
fi
# Prefixed with the repo root so this works from any directory.
local paths=()
local file
for file in "${files[@]}"; do
paths+=("${REPO_ROOT}/${file}")
done
"$PYTHON" "$CHECKER" --files "${paths[@]}"
;;
*)
"$PYTHON" "$CHECKER" --files "$@"
;;
esac
}
main "$@"

158
scripts/e2e.sh
View File

@ -13,21 +13,11 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly SCRIPT_DIR
REPO_ROOT="$(dirname "$SCRIPT_DIR")"
# The post-replay assertions, split out at the 250-line cap. Sourced
# rather than executed, because they run against the same fake server and
# the same bot process the scenarios used -- and because the split lost
# this line once already, which left every one of them silently unrun and
# the whole script exiting 127.
# shellcheck source=scripts/e2e_proofs.sh
source "$SCRIPT_DIR/e2e_proofs.sh"
readonly REPO_ROOT
readonly PYTHON="${PYTHON:-$REPO_ROOT/.venv/bin/python}"
readonly FAKE_PORT="${FAKE_PORT:-8099}"
readonly BOT_PORT="${BOT_PORT:-8098}"
readonly MANAGE_PORT="${MANAGE_PORT:-8097}"
readonly FAKE_URL="http://127.0.0.1:${FAKE_PORT}"
readonly MANAGE_URL="http://127.0.0.1:${MANAGE_PORT}"
# Both spellings of one group, exactly as the real API reports them:
# /v1/groups returns the `group.<base64>` form (what .env holds), while an
@ -49,7 +39,6 @@ readonly OTHER_GROUP="group.e2e.other=="
FAKE_PID=""
BOT_PID=""
MANAGE_PID=""
TEMP_DIR=""
cleanup() {
@ -59,9 +48,6 @@ cleanup() {
if [[ -n "$BOT_PID" ]]; then
kill "$BOT_PID" 2>/dev/null || true
fi
if [[ -n "$MANAGE_PID" ]]; then
kill "$MANAGE_PID" 2>/dev/null || true
fi
if [[ -n "$FAKE_PID" ]]; then
kill "$FAKE_PID" 2>/dev/null || true
fi
@ -101,58 +87,6 @@ wait_for_subscription() {
done
}
# Drive the fake through these scenario files, against the bot booted above.
# One place for the group map, so a proof that injects after the main replay
# resolves the same symbolic names the scenarios do.
replay() {
"$PYTHON" -m fake_signal.cli "$@" \
--settle "${E2E_SETTLE:-0.15}" \
--base-url "$FAKE_URL" \
--group "watched=${WATCHED_GROUP_INTERNAL}" \
--group "reply=${WATCHED_GROUP}" \
--group "second=${SECOND_GROUP_INTERNAL}" \
--group "second_reply=${SECOND_GROUP}" \
--group "third=${THIRD_GROUP_INTERNAL}" \
--group "third_reply=${THIRD_GROUP}" \
--group "fourth=${FOURTH_GROUP_INTERNAL}" \
--group "fourth_reply=${FOURTH_GROUP}" \
--group "other=${OTHER_GROUP}"
}
# The bot and the management page under one environment: the page approves
# into the album the bot reads, and asks the bot at BIND_PORT to rebuild its
# table -- a path that differed between them would prove nothing.
# GROUP_IDS and TRUSTED_INVITERS deliberately empty: the bot discovers its
# groups from the account, which is the configuration real deployments use.
# Set rather than merely unset, because load_dotenv() would otherwise let
# the developer's own .env decide what this run watches -- and a
# TRUSTED_INVITERS there would refuse every fixture group. The gate itself
# is covered in tests/test_trust.py.
# exec, because this is backgrounded: without it $! is a subshell, and the
# kill in cleanup would leave the real process running after the run.
with_bot_env() {
exec env -u CAT_API \
PHONE_NUMBER="+15550000000" \
GROUP_IDS="" \
TRUSTED_INVITERS="" \
API_BASE_URL="$FAKE_URL" \
DATABASE_PATH="${TEMP_DIR}/e2e.sqlite3" \
CAT_API_URL="${FAKE_URL}/_fixtures/cat-api" \
DOG_API_URL="${FAKE_URL}/_fixtures/dog-api" \
GECKO_API_URL="${FAKE_URL}/_fixtures/gecko-api" \
COMMONS_API_URL="${FAKE_URL}/_fixtures/commons-api" \
POEM_TEXT_URL="${FAKE_URL}/_fixtures/poem-text?slug={slug}" \
PORTRAIT_CACHE_DIR="${TEMP_DIR}/portraits" \
PHOTO_PACK_DIR="${TEMP_DIR}/pack" \
CUSTOM_ALIASES_FILE="${TEMP_DIR}/aliases_custom.json" \
FOOD_LOG_FILE="${TEMP_DIR}/logs/food_log.json" \
WORKOUT_LOG_FILE="${TEMP_DIR}/logs/workout_log.json" \
BIND_PORT="$BOT_PORT" \
SILENCE_SECONDS=0 \
LIVENESS_TICK_SECONDS=1 \
"$@"
}
validate_requirements() {
if [[ ! -x "$PYTHON" ]]; then
echo "Error: no interpreter at $PYTHON. Run: uv venv .venv && uv pip install -e '.[dev]'" >&2
@ -164,7 +98,6 @@ validate_requirements() {
main() {
validate_requirements
TEMP_DIR="$(mktemp -d)"
STARTED_AT="$(date +%s)"
echo "== booting the fake Signal server on :${FAKE_PORT}"
export FAKE_PORT
@ -181,38 +114,81 @@ main() {
-H 'Content-Type: application/json' \
-d "[{\"id\":\"${WATCHED_GROUP}\",\"internal_id\":\"${WATCHED_GROUP_INTERNAL}\",\"name\":\"e2e\",\"member\":true,\"joinable\":true},{\"id\":\"${SECOND_GROUP}\",\"internal_id\":\"${SECOND_GROUP_INTERNAL}\",\"name\":\"e2e-second\",\"member\":true,\"joinable\":true}]" >/dev/null
# The album is private and lives outside this repository, so the run
# builds a synthetic one: three invented people, a combination and a
# video, scoped to the watched group so the scoping scenario can prove
# it does not exist in the second one.
echo "== writing a synthetic photo pack"
"$PYTHON" "$REPO_ROOT/tool/e2e_pack.py" --dir "${TEMP_DIR}/pack" --group "$WATCHED_GROUP"
# The food and workout logs are the operator's day, kept by other
# programs; the run gets one invented entry of each, dated today.
"$PYTHON" "$REPO_ROOT/tool/e2e_logs.py" --dir "${TEMP_DIR}/logs"
echo "== generating media for !dodaj (real exif, never committed)"
"$PYTHON" "$REPO_ROOT/tool/e2e_media.py" --upload "$FAKE_URL"
# GROUP_IDS and TRUSTED_INVITERS deliberately empty: the bot discovers
# its groups from the account, which is the configuration real
# deployments use. Set rather than merely unset, because load_dotenv()
# would otherwise let the developer's own .env decide what this run
# watches -- and a TRUSTED_INVITERS there would refuse every fixture
# group. The gate itself is covered in tests/test_trust.py.
echo "== booting the bot on :${BOT_PORT}, pointed at the fake"
with_bot_env "$PYTHON" -m signal_bot &
env -u CAT_API \
PHONE_NUMBER="+15550000000" \
GROUP_IDS="" \
TRUSTED_INVITERS="" \
API_BASE_URL="$FAKE_URL" \
DATABASE_PATH="${TEMP_DIR}/e2e.sqlite3" \
CAT_API_URL="${FAKE_URL}/_fixtures/cat-api" \
DOG_API_URL="${FAKE_URL}/_fixtures/dog-api" \
GECKO_API_URL="${FAKE_URL}/_fixtures/gecko-api" \
COMMONS_API_URL="${FAKE_URL}/_fixtures/commons-api" \
PORTRAIT_CACHE_DIR="${TEMP_DIR}/portraits" \
BIND_PORT="$BOT_PORT" \
SILENCE_SECONDS=0 \
"$PYTHON" -m signal_bot &
BOT_PID=$!
echo "== booting the management page on :${MANAGE_PORT}, pointed at the bot"
with_bot_env "$PYTHON" "$REPO_ROOT/tool/web.py" \
--dir "${TEMP_DIR}/pack" --port "$MANAGE_PORT" --no-open &
MANAGE_PID=$!
wait_for_url "http://127.0.0.1:${BOT_PORT}/healthz" "bot"
wait_for_url "${MANAGE_URL}/api/album/pending" "management page"
wait_for_subscription
echo "== replaying scenarios"
replay "$REPO_ROOT"/fake_signal/scenarios/*.yaml
"$PYTHON" -m fake_signal.cli \
"$REPO_ROOT"/fake_signal/scenarios/*.yaml \
--base-url "$FAKE_URL" \
--group "watched=${WATCHED_GROUP_INTERNAL}" \
--group "reply=${WATCHED_GROUP}" \
--group "second=${SECOND_GROUP_INTERNAL}" \
--group "second_reply=${SECOND_GROUP}" \
--group "third=${THIRD_GROUP_INTERNAL}" \
--group "third_reply=${THIRD_GROUP}" \
--group "fourth=${FOURTH_GROUP_INTERNAL}" \
--group "fourth_reply=${FOURTH_GROUP}" \
--group "other=${OTHER_GROUP}"
prove_the_add_photo_queue
prove_the_table_reloads_without_a_restart
prove_a_switch_is_live_without_a_reload
prove_an_alias_is_live_after_a_reload
prove_the_deafness_gate
enforce_the_time_budget "$STARTED_AT"
}
# The gate that did not exist on 2026-08-29, proven against the real process:
# with signal-cli unable to reach Signal, /healthz must go red rather than
# keep answering 200 while nothing arrives.
prove_the_deafness_gate() {
local health="http://127.0.0.1:${BOT_PORT}/healthz"
echo "== cutting the fake's connection to Signal"
curl -sf -X POST "${FAKE_URL}/_control/connection" \
-H 'Content-Type: application/json' -d '{"alive": false}' >/dev/null
local code="" waited=0
while ((waited < 150)); do
code="$(curl -s -o /dev/null -w '%{http_code}' "$health")"
[[ "$code" == "503" ]] && break
sleep 1
waited=$((waited + 1))
done
[[ "$code" == "503" ]] || die "/healthz still answers ${code} with the connection cut"
curl -s "$health" | grep -q "Closed unexpectedly" ||
die "/healthz went red without saying why"
echo "== /healthz turned 503 and named the reason after ${waited}s"
echo "== restoring the connection"
curl -sf -X POST "${FAKE_URL}/_control/connection" \
-H 'Content-Type: application/json' -d '{"alive": true}' >/dev/null
waited=0
while ((waited < 150)); do
code="$(curl -s -o /dev/null -w '%{http_code}' "$health")"
[[ "$code" == "200" ]] && break
sleep 1
waited=$((waited + 1))
done
[[ "$code" == "200" ]] || die "/healthz stayed ${code} after the connection came back"
echo "== /healthz recovered on its own after ${waited}s"
}
main "$@"

201
scripts/e2e_proofs.sh
View File

@ -1,201 +0,0 @@
#!/bin/bash
# ============================================================================
# The assertions scripts/e2e.sh makes after the scenarios have replayed.
#
# Split out of e2e.sh at the 250-line cap. A filing change, not an interface
# change: these are sourced, so they still run against the same fake server
# and the same bot process the scenarios used.
# ============================================================================
# The gate that did not exist on 2026-08-29, proven against the real process:
# with signal-cli unable to reach Signal, /healthz must go red rather than
# keep answering 200 while nothing arrives.
prove_the_deafness_gate() {
local health="http://127.0.0.1:${BOT_PORT}/healthz"
echo "== cutting the fake's connection to Signal"
curl -sf -X POST "${FAKE_URL}/_control/connection" \
-H 'Content-Type: application/json' -d '{"alive": false}' >/dev/null
local code="" waited=0
while ((waited < 150)); do
code="$(curl -s -o /dev/null -w '%{http_code}' "$health")"
[[ "$code" == "503" ]] && break
sleep 1
waited=$((waited + 1))
done
[[ "$code" == "503" ]] || die "/healthz still answers ${code} with the connection cut"
curl -s "$health" | grep -q "Closed unexpectedly" ||
die "/healthz went red without saying why"
echo "== /healthz turned 503 and named the reason after ${waited}s"
echo "== restoring the connection"
curl -sf -X POST "${FAKE_URL}/_control/connection" \
-H 'Content-Type: application/json' -d '{"alive": true}' >/dev/null
waited=0
while ((waited < 150)); do
code="$(curl -s -o /dev/null -w '%{http_code}' "$health")"
[[ "$code" == "200" ]] && break
sleep 1
waited=$((waited + 1))
done
[[ "$code" == "200" ]] || die "/healthz stayed ${code} after the connection came back"
echo "== /healthz recovered on its own after ${waited}s"
}
# A person tagged while the bot is up becomes a command after POST /_reload,
# with no restart: the same process that just answered "Nie znam !delta"
# posts delta's photo a moment later. The scenarios live next to the main
# set but outside its glob, because the second one is only true after the
# reload that this proof performs between them.
prove_the_table_reloads_without_a_restart() {
local proofs="${REPO_ROOT}/fake_signal/proofs"
echo "== before the reload, !delta is nobody"
replay "${proofs}/reload_before.yaml"
echo "== tagging a fifth person into the pack, then scripts/reload.sh"
"$PYTHON" "$REPO_ROOT/tool/e2e_pack.py" \
--dir "${TEMP_DIR}/pack" --group "$WATCHED_GROUP" --extra delta >/dev/null
# The operator's script, not a bare curl: this is the one place it runs
# against a real process in CI, so its exit code is the proof.
BIND_PORT="$BOT_PORT" "$REPO_ROOT/scripts/reload.sh" ||
die "scripts/reload.sh did not get the new table live"
echo "== after the reload, !delta posts a photo"
replay "${proofs}/reload_after.yaml"
}
# A switch thrown from the management page is read on the very next attempt:
# no reload, no restart. The PUT here is what the page's Save button sends,
# aimed at the bot directly; the two scenarios bracket it the same way the
# reload pair does. Not a timed switch, because a proof that depends on the
# clock has to wait on it, and the unit tests already cover the deadline.
prove_a_switch_is_live_without_a_reload() {
local proofs="${REPO_ROOT}/fake_signal/proofs"
local switch="http://127.0.0.1:${BOT_PORT}/_commands/cat"
echo "== switching !kot off"
curl -sf -X PUT "$switch" -H 'Content-Type: application/json' \
-d '{"enabled": false}' | grep -q '"enabled":false' ||
die "PUT /_commands/cat did not report the command off"
replay "${proofs}/switch_off.yaml"
echo "== switching !kot back on"
curl -sf -X PUT "$switch" -H 'Content-Type: application/json' \
-d '{"enabled": true}' | grep -q '"enabled":true' ||
die "PUT /_commands/cat did not report the command on"
replay "${proofs}/switch_on.yaml"
}
# The operator's alias file is data the table is built from, so it takes
# the same path as a tagged person: written, then scripts/reload.sh. The
# file is the one CUSTOM_ALIASES_FILE points the bot at, never the
# operator's own; a name the table does not hold is what the second half
# proves the bot refuses, with the old table still answering.
prove_an_alias_is_live_after_a_reload() {
local proofs="${REPO_ROOT}/fake_signal/proofs"
local file="${TEMP_DIR}/aliases_custom.json"
echo "== before the alias, !e2ekot is nobody"
replay "${proofs}/alias_before.yaml"
echo "== writing the operator's alias file, then scripts/reload.sh"
printf '{"cat": ["!e2ekot"]}\n' > "$file"
BIND_PORT="$BOT_PORT" "$REPO_ROOT/scripts/reload.sh" ||
die "scripts/reload.sh did not get the alias live"
replay "${proofs}/alias_after.yaml"
echo "== an alias for a command that does not exist keeps the old table"
printf '{"cat": ["!e2ekot"], "nosuch": ["!nic"]}\n' > "$file"
local answer
answer="$(curl -s -w '%{http_code}' -X POST "http://127.0.0.1:${BOT_PORT}/_reload")"
[[ "$answer" == *nosuch*409 ]] ||
die "a bad alias file was not kept out with 409 and its name: ${answer}"
replay "${proofs}/alias_kept.yaml"
printf '{"cat": ["!e2ekot"]}\n' > "$file"
}
# A budget, not a benchmark. This run was four minutes until the liveness
# tick became configurable, and nothing announced the cost -- slower is the
# one regression a green suite never reports. Exceeding it fails the run;
# whoever exceeded it decides whether the check earned its seconds or the
# budget moves. E2E_BUDGET_SECONDS raises it for a loaded machine.
# What the !dodaj replies cannot prove on their own: every queued file was
# stripped before it waited anywhere, a typed label reached nothing but the
# queue, the refused and out-of-scope sends wrote nothing -- and then the one
# door into the album, taken through the management page, after which the
# bot answers with the photo and recognises a second copy of it.
prove_the_add_photo_queue() {
local pack="${TEMP_DIR}/pack"
echo "== checking every queued file was stripped before it waited"
"$PYTHON" "$REPO_ROOT/tool/e2e_media.py" --verify-pack "$pack"
echo "== checking a typed label reached nothing but the queue"
if [[ -e "${pack}/custom.json" ]]; then
echo "Error: custom.json was written before anything was approved" >&2
exit 1
fi
if grep -rq "piesek" "$pack"; then
echo "Error: a label survived a message that carried no attachment" >&2
exit 1
fi
echo "== checking exactly the accepted sends are waiting"
local queued
queued="$(find "${pack}/pending" -type f ! -name '*.json' | wc -l)"
if [[ "$queued" != "5" ]]; then
echo "Error: expected 5 queued files (person, custom, heic, svg, video), found ${queued}" >&2
exit 1
fi
echo "== checking signal-cli's own copies were deleted, and only those"
local left
left="$(curl -sf "${FAKE_URL}/_control/attachments")"
for taken in att-person att-custom att-heic att-svg att-video att-junk; do
if [[ "$left" == *"$taken"* ]]; then
echo "Error: ${taken} is still in the container store: ${left}" >&2
exit 1
fi
done
# Out of scope means never fetched: the second group's file is untouched.
if [[ "$left" != *"att-second"* ]]; then
echo "Error: a send in a group without the album was fetched: ${left}" >&2
exit 1
fi
echo "== approving the queue through the management page"
"$PYTHON" "$REPO_ROOT/tool/e2e_approve.py" --manage-url "$MANAGE_URL"
echo "== checking the album took them whole, and the queue is empty"
"$PYTHON" "$REPO_ROOT/tool/e2e_media.py" --verify-pack "$pack"
if [[ -n "$(ls -A "${pack}/pending")" ]]; then
echo "Error: the queue is not empty after approving everything" >&2
exit 1
fi
if grep -qE '"(wakacje|piesek)"[[:space:]]*:' "${pack}/custom.json"; then
echo "Error: a typed label became a key in custom.json" >&2
exit 1
fi
if ! grep -q '"approved_at"' "${pack}/adds.json"; then
echo "Error: adds.json does not record the approval" >&2
exit 1
fi
echo "== replaying what only an approved photo can answer"
"$PYTHON" "$REPO_ROOT/tool/e2e_media.py" --upload "$FAKE_URL" >/dev/null
replay "$REPO_ROOT"/fake_signal/approved/*.yaml
}
enforce_the_time_budget() {
local started="$1"
local budget="${E2E_BUDGET_SECONDS:-40}"
local elapsed=$(( $(date +%s) - started ))
echo "== end to end took ${elapsed}s (budget ${budget}s)"
if ((elapsed > budget)); then
echo "Error: the offline run took ${elapsed}s, over its ${budget}s budget." >&2
echo " Either the new check earns its seconds and the budget moves," >&2
echo " or something got slower and this is the report of it." >&2
exit 1
fi
}

49
scripts/ensure_venv.sh
View File

@ -1,49 +0,0 @@
#!/bin/bash
# ============================================================================
# Make sure .venv exists and has this project installed in it.
#
# run.sh used to answer a missing venv with a message telling you to type
# "uv venv .venv && uv pip install -e '.[dev]'" yourself, and tool/tagger.py
# did not even do that -- it failed with ModuleNotFoundError: fastapi. Both
# are the same rule broken: a script owns its dependencies.
#
# Sourced or executed. Idempotent: the common case is two file tests and no
# subprocess at all, so entry points can call it unconditionally.
# ============================================================================
set -euo pipefail
ensure_venv() {
local repo_root="$1"
local venv="${repo_root}/.venv"
local python="${venv}/bin/python"
# Already good: the interpreter exists and the project imports from it.
if [[ -x "$python" ]] && "$python" -c 'import fastapi, signal_bot' 2>/dev/null; then
return 0
fi
echo "== setting up ${venv} (one-off)" >&2
if [[ ! -x "$python" ]]; then
if command -v uv >/dev/null; then
uv venv "$venv"
elif ! python3 -m venv "$venv"; then
echo "Error: could not create a virtualenv at ${venv}" >&2
echo " python3 -m venv failed; on Arch: sudo pacman -S python" >&2
return 1
fi
fi
# Editable, so a later git pull updates what the tagger actually runs.
if command -v uv >/dev/null; then
VIRTUAL_ENV="$venv" uv pip install -e "${repo_root}[dev]"
else
"$python" -m pip install -e "${repo_root}[dev]"
fi
}
# Executed rather than sourced: do the thing, for callers that are not bash.
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
ensure_venv "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
fi

62
scripts/install_service.sh
View File

@ -47,70 +47,8 @@ bot_port() {
printf '%s' "${port:-$DEFAULT_PORT}"
}
# exiftool is what !add_photo strips uploads with, and the strip is not
# optional: without it every upload is refused rather than stored dirty.
# Installed here rather than documented, per the repo's own rule that a
# script owns its dependencies.
ensure_exiftool() {
if ! command -v exiftool >/dev/null; then
echo "Installing perl-image-exiftool (needed by !add_photo)..."
if ! sudo pacman -S --needed --noconfirm perl-image-exiftool; then
die "could not install perl-image-exiftool; run: sudo pacman -S perl-image-exiftool"
fi
fi
# `command -v` answers for *this shell*, which is not who has to find it.
# Arch installs to /usr/bin/vendor_perl, on an interactive PATH and not on
# a systemd unit's -- so on 2026-08-31 the bot refused every upload while
# exiftool sat plainly on the installer's PATH. Ask the bot's own resolver.
if ! env -i "PATH=/usr/local/bin:/usr/bin" "${REPO_ROOT}/.venv/bin/python" -c \
'import sys; from signal_bot.media_strip import locate; sys.exit(0 if locate() else 1)'; then
die "exiftool is installed but signal_bot.media_strip.locate() cannot find it under the unit's PATH"
fi
}
# ffmpeg drops the metadata *tracks* exiftool cannot see. An iPhone clip
# carries mebx timed-metadata holding per-frame face bounding boxes and a
# face id, and those survive a complete "-all= -time:all=" pass because they
# live in mdat as opaque samples, not as tags. Without ffmpeg every video is
# refused rather than stored with them.
ensure_ffmpeg() {
if ! command -v ffmpeg >/dev/null || ! command -v ffprobe >/dev/null; then
echo "Installing ffmpeg (needed to store video)..."
if ! sudo pacman -S --needed --noconfirm ffmpeg; then
die "could not install ffmpeg; run: sudo pacman -S ffmpeg"
fi
fi
# Same trap as exiftool: ask the bot's own resolver, under the unit's PATH.
if ! env -i "PATH=/usr/local/bin:/usr/bin" "${REPO_ROOT}/.venv/bin/python" -c \
'import sys; from signal_bot.media_tracks import locate; sys.exit(0 if locate("ffmpeg") and locate("ffprobe") else 1)'; then
die "ffmpeg is installed but signal_bot.media_tracks.locate() cannot find it under the unit's PATH"
fi
}
# The management page is built, not committed: dist/ is a build output and
# this repository does not carry those. Verified by looking at the artefact
# rather than trusting the exit code, the same way exiftool is.
build_web() {
local web="${REPO_ROOT}/web"
if ! command -v pnpm >/dev/null; then
echo "Installing pnpm (the management page is built with it)..."
if ! sudo pacman -S --needed --noconfirm pnpm; then
die "could not install pnpm; run: sudo pacman -S pnpm"
fi
fi
command -v node >/dev/null || die "node not found; the page needs node >= 24"
echo "Building the management page..."
(cd "$web" && pnpm install --frozen-lockfile && pnpm build) ||
die "pnpm build failed in ${web}"
[[ -f "${web}/dist/index.html" ]] ||
die "pnpm build reported success but ${web}/dist/index.html is not there"
}
validate_requirements() {
command -v systemctl >/dev/null || die "systemctl not found; this needs systemd"
ensure_exiftool
ensure_ffmpeg
build_web
[[ -f "${REPO_ROOT}/.env" ]] ||
die "no ${REPO_ROOT}/.env -- run scripts/register_bot.sh first"
[[ -x "${REPO_ROOT}/run.sh" ]] || die "no executable ${REPO_ROOT}/run.sh"

219
scripts/personal_data_sources.py
View File

@ -1,219 +0,0 @@
#!/usr/bin/env python3
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Where the forbidden words come from -- all of it outside this repository.
Split out of check_personal_data.py at the 250-line cap, and the seam is the
honest one: this half knows the private data, that half knows nothing but how
to search. A list of real names kept *in* the repository to check against
would be the leak the check exists to prevent, so nothing here is ever
written down -- it is read at run time from the photo pack, the message
database and the local .env, and on a machine with none of those it simply
comes back empty.
"""
from __future__ import annotations
import base64
import json
import os
import sqlite3
from pathlib import Path
from typing import Any, Final
REPO: Final = Path(__file__).resolve().parent.parent
# Three, because real first names are three letters long and one of this
# group's is. Two would match inside half the words in the repository; three
# needs a handful of allowlist entries for the given names that are also
# popes, which is a cost worth paying to catch a real one.
MIN_NAME: Final = 3
# How much of a group id has to appear before it counts as that group. Eight
# base64 characters are 48 bits: unique enough that a coincidence is not
# worth worrying about, short enough to catch a truncated quotation.
PREFIX: Final = 8
def _pack_names(directory: Path) -> set[str]:
"""Every trigger word in a photo pack manifest -- i.e. every name."""
document = _pack_document(directory / "people.json")
entries = [*document.get("people", []), *document.get("combos", [])]
return {
str(trigger).lstrip("!").strip()
for entry in entries
for trigger in entry.get("triggers", [])
}
def _custom_labels(directory: Path) -> set[str]:
"""Every label !add_photo minted or proposed -- names too, however they got there.
Read separately from _pack_names because they are a different kind of
thing. A people.json trigger is curated by the tagger; a custom label is
free text somebody typed into a group chat, and treating arbitrary free
text as a forbidden literal is what lets one message wedge this gate
permanently. See _repo_vocabulary for the half that makes that safe.
The approval queue's sidecars (``pending/*.json``) count too: a label
waiting there is the same typed text, one click earlier.
"""
labels: set[str] = set()
for entry in _pack_document(directory / "custom.json").values():
if isinstance(entry, list):
labels |= {str(label).lstrip("!").strip() for label in entry}
for sidecar in sorted((directory / "pending").glob("*.json")):
proposed = _pack_document(sidecar).get("labels")
if isinstance(proposed, list):
labels |= {str(label).lstrip("!").strip() for label in proposed}
return labels
def _pack_document(path: Path) -> dict[str, Any]:
"""One album manifest, or an empty mapping when it is absent or broken."""
if not path.is_file():
return {}
try:
document = json.loads(path.read_text(encoding="utf-8"))
except OSError, json.JSONDecodeError:
return {}
return document if isinstance(document, dict) else {}
def _database_names(path: Path) -> set[str]:
"""Display names of everybody the bot has ever seen speak."""
if not path.is_file():
return set()
try:
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
except sqlite3.Error:
return set()
# Literal statements, not a table name interpolated into one: a
# formatted query is a SQL-injection shape even when every input is a
# constant, and this repository takes no lint suppressions.
queries = (
"SELECT DISTINCT display_name FROM message_counts",
"SELECT DISTINCT display_name FROM group_people",
"SELECT DISTINCT display_name FROM message_events",
)
found: set[str] = set()
with connection:
for query in queries:
try:
rows = connection.execute(query)
except sqlite3.Error:
continue
found.update(str(row[0]) for row in rows if row[0])
connection.close()
return found
def _database_groups(path: Path) -> set[str]:
"""Every real group id the bot has seen, in both spellings.
Both, because they are the same group written two ways: /v1/groups
reports "group.<base64>" and an inbound envelope carries the id that
base64 decodes to. A check that knew only the first would have missed the
one that was sitting in tests/test_config.py.
"""
if not path.is_file():
return set()
try:
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
except sqlite3.Error:
return set()
queries = (
"SELECT DISTINCT group_id FROM group_meta",
"SELECT DISTINCT group_id FROM message_events",
)
found: set[str] = set()
with connection:
for query in queries:
try:
rows = connection.execute(query)
except sqlite3.Error:
continue
found.update(str(row[0]) for row in rows if row[0])
connection.close()
inner: set[str] = set()
for group in found:
payload = group.removeprefix("group.")
# The leading run as well as the whole id: a log line quoted into a
# commit message says "group.<first few chars>..." and stops, and
# matching only the full id let exactly that through once.
inner.add(payload[:PREFIX])
try:
inner.add(base64.b64decode(payload, validate=True).decode("utf-8"))
except ValueError, UnicodeDecodeError:
continue
return found | inner
def _env_values(path: Path) -> set[str]:
"""Phone numbers, group ids and inviters from the deployment's .env."""
if not path.is_file():
return set()
interesting = {"PHONE_NUMBER", "GROUP_IDS", "GROUP_ID", "TRUSTED_INVITERS"}
found: set[str] = set()
for line in path.read_text(encoding="utf-8").splitlines():
key, _, value = line.partition("=")
if key.strip() in interesting:
found.update(part.strip() for part in value.split(",") if part.strip())
return found
def forbidden(permitted: set[str]) -> tuple[set[str], list[str]]:
"""Every literal that must not be committed, and where they came from.
The sources are all outside the repository on purpose. A list of real
names kept *in* the repository to check against would be the leak it is
meant to prevent.
"""
pack = os.environ.get("PHOTO_PACK_DIR", "").strip()
directory = (
Path(pack)
if pack
else Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
/ "signal-bot"
/ "pack"
)
sources: list[str] = []
words: set[str] = set()
for label, found in (
(f"pack manifest ({directory})", _pack_names(directory)),
(f"pack custom tags ({directory})", _custom_labels(directory) - _repo_vocabulary()),
("message database", _database_names(REPO / "signal_bot.sqlite3")),
("group ids", _database_groups(REPO / "signal_bot.sqlite3")),
(".env", _env_values(REPO / ".env")),
):
if found:
sources.append(label)
words |= found
return {
word.lower() for word in words if len(word) >= MIN_NAME and word.lower() not in permitted
}, sources
def _repo_vocabulary() -> set[str]:
"""Words this public repository already contains as its own command names.
Subtracted from the custom labels, and only from those. A word that is
committed here *already* cannot be a private name this gate is stopping
from getting out -- it is out. Without the subtraction, one member
sending "!add_photo kot <photo>" would mint the label "kot", and the gate
would then find "kot" in aliases.json and refuse every commit from then
on, fixable only by hand-editing an album the committer cannot see.
people.json triggers are deliberately *not* filtered this way: those are
curated real names, and one of them turning up in the repository is
exactly the leak worth failing on.
"""
try:
document = json.loads((REPO / "signal_bot" / "aliases.json").read_text(encoding="utf-8"))
except OSError, json.JSONDecodeError:
return set()
return {
str(trigger).lstrip("!").strip().lower()
for name, triggers in document.items()
if not name.startswith("_") and isinstance(triggers, list)
for trigger in triggers
}

121
scripts/reload.sh
View File

@ -1,121 +0,0 @@
#!/bin/bash
# ============================================================================
# Make the running bot pick up a change, and prove that it did.
#
# Two kinds of change, two mechanisms:
#
# scripts/reload.sh [cmd] data changed (aliases, an album, a
# command switched off): POST /_reload,
# the table is rebuilt in place, nothing
# restarts and no frame is lost.
# scripts/reload.sh --code [cmd] code changed: restart the unit, wait for
# /healthz to go green again. Lossless too
# (signal-cli replays what arrived), but
# a deploy step -- so it is the flag, not
# the default.
#
# With [cmd], the bot then types that command at itself in the test group,
# so "reloaded" means "answered", not "returned 200". E.g.
#
# scripts/reload.sh '!nowaosoba'
# scripts/reload.sh --code '!pomoc'
# ============================================================================
set -euo pipefail
SCRIPT_NAME="$(basename "$0")"
readonly SCRIPT_NAME
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly SCRIPT_DIR
readonly BOT="http://127.0.0.1:${BIND_PORT:-8137}"
readonly UNIT="signal-bot"
readonly TEST_GROUP="${RELOAD_GROUP:-testbot}"
RESTART=0
PROBE=""
usage() {
echo "Usage: $SCRIPT_NAME [--code] [command-to-selftest]"
echo " --code restart the unit (code changed) instead of rebuilding the table"
exit 0
}
validate_requirements() {
if ! command -v jq >/dev/null; then
echo "Error: jq is required" >&2
exit 1
fi
}
wait_healthy() {
local attempt=0
# 150 x 0.4 s: a restart is ~1 s, but signal-cli can take a while to
# accept the websocket again after its own reconnect.
until [[ "$(curl -s "${BOT}/healthz" | jq -r '.status // "down"')" == "ok" ]]; do
attempt=$((attempt + 1))
if [[ $attempt -ge 150 ]]; then
echo "Error: ${BOT}/healthz never reported ok" >&2
curl -s "${BOT}/healthz" >&2 || true
exit 1
fi
sleep 0.4
done
}
reload_data() {
local reply status
reply="$(curl -s -X POST "${BOT}/_reload")"
status="$(jq -r '.status // "?"' <<<"$reply")"
case "$status" in
reloaded)
echo "reloaded: $(jq -r '.commands' <<<"$reply") commands live"
;;
kept)
# The old table is still answering; the new data is what is broken.
echo "Error: reload refused, previous table kept: $(jq -r '.reason' <<<"$reply")" >&2
exit 1
;;
*)
echo "Error: unexpected answer from ${BOT}/_reload: ${reply}" >&2
exit 1
;;
esac
}
restart_code() {
systemctl --user restart "$UNIT"
wait_healthy
echo "restarted: $(curl -s "${BOT}/healthz" | jq -r '"\(.version) \(.pope)"')"
}
main() {
validate_requirements
if [[ $RESTART -eq 1 ]]; then
restart_code
else
wait_healthy
reload_data
fi
if [[ -n "$PROBE" ]]; then
"${SCRIPT_DIR}/selftest.sh" "$TEST_GROUP" "$PROBE"
fi
}
while [[ $# -gt 0 ]]; do
case $1 in
--code)
RESTART=1
shift
;;
-h | --help)
usage
;;
*)
PROBE="$1"
shift
;;
esac
done
main

4
scripts/signal-bot.service
View File

@ -44,10 +44,6 @@ PrivateTmp=true
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=@REPO_ROOT@
# The photo album. ProtectHome=read-only covers it too, and !add_photo
# writes there -- media, tags.json, custom.json and adds.json. The pack
# does not move; this is the one path that has to become writable.
ReadWritePaths=%h/.local/share/signal-bot
ProtectKernelTunables=true
ProtectControlGroups=true
RestrictSUIDSGID=true

70
scripts/signal_api_watchdog.sh
View File

@ -28,16 +28,9 @@ readonly FAILURES_BEFORE_RESTART="${FAILURES_BEFORE_RESTART:-2}"
readonly MAX_RESTARTS="${MAX_RESTARTS:-3}"
readonly RESTART_WINDOW="${RESTART_WINDOW:-21600}" # 6 hours
readonly SETTLE_SECONDS="${SETTLE_SECONDS:-45}"
# always: every transition pops. escalation: only the give-up does. never: none.
# Default escalation, because the routine restart/recovery pair is exactly the
# noise this knob exists to stop; journald still gets all of it either way.
readonly NOTIFY_MODE="${WATCHDOG_NOTIFY:-escalation}"
readonly NOTIFY_EXPIRE_MS="${WATCHDOG_NOTIFY_EXPIRE_MS:-20000}"
readonly FAILURE_FILE="${STATE_DIR}/consecutive_failures"
readonly RESTART_FILE="${STATE_DIR}/restarts"
readonly NOTIFY_ID_FILE="${STATE_DIR}/notify_id"
readonly ESCALATED_FILE="${STATE_DIR}/escalated_at"
usage() {
echo "Usage: $SCRIPT_NAME"
@ -52,59 +45,12 @@ die() {
exit 1
}
# The escalation repeats on every firing for as long as the window lasts. The
# journal wants all of it; the desktop wants one box. So the stamp gates the
# POPUP only -- never the log lines, which is why the give-up text still
# appears on stdout for every check that reaches it.
escalation_is_new() {
local now stamp=0
now="$(date +%s)"
if [[ -f "$ESCALATED_FILE" ]]; then
read -r stamp <"$ESCALATED_FILE" || true
[[ "$stamp" =~ ^[0-9]+$ ]] || stamp=0
fi
if ((now - stamp < RESTART_WINDOW)); then
return 1
fi
printf '%s\n' "$now" >"$ESCALATED_FILE"
}
# One desktop slot, reused. Without --replace-id every firing stacked another
# popup that xfce4-notifyd never expires, so a single outage left a screenful
# of identical boxes to dismiss by hand -- 297 of them in one day.
notify() {
local urgency="$1" message="$2" previous=0 id
case "$NOTIFY_MODE" in
never) return 0 ;;
escalation)
if [[ "$urgency" != critical ]]; then
return 0
fi
;;
*) ;;
esac
if [[ "$urgency" == critical ]] && ! escalation_is_new; then
return 0
fi
if [[ -f "$NOTIFY_ID_FILE" ]]; then
read -r previous <"$NOTIFY_ID_FILE" || true
[[ "$previous" =~ ^[0-9]+$ ]] || previous=0
fi
id="$(notify-send --print-id --replace-id="$previous" --urgency="$urgency" \
--expire-time="$NOTIFY_EXPIRE_MS" --app-name=signal-bot \
"signal-bot" "$message" 2>/dev/null)" || return 0
if [[ "$id" =~ ^[0-9]+$ ]]; then
printf '%s\n' "$id" >"$NOTIFY_ID_FILE"
fi
}
# journald and stdout always; the desktop only when notify() agrees. The broken
# path IS Signal, so the bot cannot tell anybody about this itself -- but it
# only needs to say it once.
# journald and the desktop, always both: the broken path IS Signal, so the
# bot cannot tell anybody about this itself.
alarm() {
logger --priority user.warning --tag signal-api-watchdog -- "$1" || true
echo "== $1"
notify "${2:-normal}" "$1"
notify-send --urgency=critical --app-name=signal-bot "signal-bot" "$1" 2>/dev/null || true
}
validate_requirements() {
@ -143,17 +89,10 @@ connection_is_alive() {
return 1
}
# Recovery re-arms the escalation: the next outage is a new event and deserves
# its own box, rather than being silenced by a stamp from the previous one.
clear_alarms() {
rm -f "$ESCALATED_FILE"
}
record_success() {
local previous
previous="$(read_count)"
printf '0\n' >"$FAILURE_FILE"
clear_alarms
if ((previous >= FAILURES_BEFORE_RESTART)); then
alarm "signal-api is reachable again after ${previous} failed probe(s)."
fi
@ -186,7 +125,7 @@ give_up() {
This is probably not transient: the account may be deregistered, or the host \
firewall may be blocking the container. Check 'docker exec ${CONTAINER} head -5 \
/var/log/signal-cli-json-rpc-1/out.log' and 'docker exec ${CONTAINER} getent hosts \
chat.signal.org'. No further restarts until the window passes." critical
chat.signal.org'. No further restarts until the window passes."
}
restart_container() {
@ -196,7 +135,6 @@ restart_container() {
sleep "$SETTLE_SECONDS"
if connection_is_alive; then
printf '0\n' >"$FAILURE_FILE"
clear_alarms
alarm "signal-api is back: the connection to Signal is alive again."
return 0
fi

94
scripts/test_watchdog.sh
View File

@ -33,8 +33,70 @@ ok() {
echo " ok: $1"
}
# shellcheck source=scripts/test_watchdog_harness.sh
source "${REPO_ROOT}/scripts/test_watchdog_harness.sh"
# Rebuild a pristine sandbox: stub binaries, stub answers, empty state.
setup() {
rm -rf "$WORK"
mkdir -p "$WORK/bin" "$WORK/state"
printf '200' >"$WORK/bot_code"
printf '200' >"$WORK/api_code"
printf '0' >"$WORK/fix_on_restart"
cat >"$WORK/bin/curl" <<EOF
#!/bin/bash
url="\${*: -1}"
case "\$url" in
*/healthz) cat "$WORK/bot_code" ;;
*) cat "$WORK/api_code" ;;
esac
EOF
cat >"$WORK/bin/docker" <<EOF
#!/bin/bash
echo "\$@" >> "$WORK/docker.log"
if [[ "\$(cat "$WORK/fix_on_restart")" == "1" ]]; then
printf '200' > "$WORK/bot_code"
printf '200' > "$WORK/api_code"
fi
EOF
chmod +x "$WORK/bin/curl" "$WORK/bin/docker"
}
run_watchdog() {
PATH="$WORK/bin:$PATH" \
PHONE_NUMBER="+15550000000" \
STATE_DIRECTORY="$WORK/state" \
SETTLE_SECONDS=0 \
FAILURES_BEFORE_RESTART=2 \
MAX_RESTARTS=2 \
RESTART_WINDOW=3600 \
"$WATCHDOG" 2>&1
}
expect_contains() {
if [[ "$1" == *"$2"* ]]; then
ok "$3"
else
fail "$3 -- got: $1"
fi
}
expect_missing() {
if [[ "$1" != *"$2"* ]]; then
ok "$3"
else
fail "$3 -- unexpectedly found $2"
fi
}
expect_file() {
local actual
actual="$(cat "$1" 2>/dev/null || echo MISSING)"
if [[ "$actual" == "$2" ]]; then
ok "$3"
else
fail "$3 -- $1 is '$actual', expected '$2'"
fi
}
test_healthy_is_free() {
echo "== a healthy bot needs no restart and no authenticated probe"
@ -46,7 +108,6 @@ test_healthy_is_free() {
else
ok "docker was never called"
fi
expect_notifications 0 "a healthy check pops nothing"
}
test_one_failure_waits() {
@ -77,7 +138,6 @@ test_confirmed_failure_restarts_and_recovers() {
expect_contains "$(cat "$WORK/docker.log")" "restart signal-api" "the container was restarted"
expect_contains "$out" "is back" "the recovery is announced"
expect_file "$WORK/state/consecutive_failures" "0" "the counter was cleared"
expect_notifications 0 "the routine restart/recovery pair stays out of the desktop"
}
test_restart_that_does_not_help() {
@ -110,31 +170,6 @@ test_gives_up_instead_of_looping() {
fi
expect_contains "$out" "still deaf" "the give-up message explains what to check"
expect_contains "$out" "out.log" "the give-up message names the log to read"
# Three firings reach give_up here; the journal got all three (asserted
# above, on the LAST run's stdout) and the desktop got one.
expect_notifications 1 "a sustained give-up window pops exactly once"
}
test_notify_modes() {
echo "== WATCHDOG_NOTIFY decides what reaches the desktop"
setup
printf '400' >"$WORK/bot_code"
printf '400' >"$WORK/api_code"
WATCHDOG_NOTIFY=never run_watchdog >/dev/null
WATCHDOG_NOTIFY=never run_watchdog >/dev/null
expect_notifications 0 "never pops nothing at all"
setup
printf '400' >"$WORK/bot_code"
printf '400' >"$WORK/api_code"
WATCHDOG_NOTIFY=always run_watchdog >/dev/null
WATCHDOG_NOTIFY=always run_watchdog >/dev/null
# Two sends -- "restarting" then "still deaf" -- collapsed by --replace-id
# into one box on screen. Counting sends is what makes the reuse visible.
expect_notifications 2 "always pops the routine pair the default suppresses"
expect_contains "$(cat "$WORK/notify.log")" "--replace-id" \
"the popup reuses one desktop slot instead of stacking"
expect_file "$WORK/state/notify_id" "4242" "the slot id is persisted for the next firing"
}
test_requires_a_number() {
@ -181,7 +216,6 @@ main() {
test_confirmed_failure_restarts_and_recovers
test_restart_that_does_not_help
test_gives_up_instead_of_looping
test_notify_modes
test_requires_a_number
echo "============================================================"

106
scripts/test_watchdog_harness.sh
View File

@ -1,106 +0,0 @@
# shellcheck shell=bash
# ============================================================================
# The sandbox and the assertions for test_watchdog.sh.
#
# Split out when the two together crossed the shared 250-line cap. Sourced,
# never executed: it defines the stub PATH the suite runs against and the
# expect_* helpers, and it owns WORK/FAILURES set by its caller.
# ============================================================================
# Rebuild a pristine sandbox: stub binaries, stub answers, empty state.
setup() {
rm -rf "$WORK"
mkdir -p "$WORK/bin" "$WORK/state"
printf '200' >"$WORK/bot_code"
printf '200' >"$WORK/api_code"
printf '0' >"$WORK/fix_on_restart"
cat >"$WORK/bin/curl" <<EOF
#!/bin/bash
url="\${*: -1}"
case "\$url" in
*/healthz) cat "$WORK/bot_code" ;;
*) cat "$WORK/api_code" ;;
esac
EOF
cat >"$WORK/bin/docker" <<EOF
#!/bin/bash
echo "\$@" >> "$WORK/docker.log"
if [[ "\$(cat "$WORK/fix_on_restart")" == "1" ]]; then
printf '200' > "$WORK/bot_code"
printf '200' > "$WORK/api_code"
fi
EOF
# Stubbed because they are NOT stubbed by default: notify-send and logger
# resolve to the real binaries, so this suite used to fire a dozen genuine
# critical popups per run -- and it runs from pre-commit, which made every
# commit spam the desktop. notify.log is now also what proves the dedup.
cat >"$WORK/bin/notify-send" <<EOF
#!/bin/bash
echo "\$@" >> "$WORK/notify.log"
echo 4242
EOF
cat >"$WORK/bin/logger" <<EOF
#!/bin/bash
echo "\$@" >> "$WORK/logger.log"
EOF
chmod +x "$WORK/bin/curl" "$WORK/bin/docker" "$WORK/bin/notify-send" "$WORK/bin/logger"
}
notify_count() {
[[ -f "$WORK/notify.log" ]] || {
printf '0'
return 0
}
grep -c '' "$WORK/notify.log"
}
expect_notifications() {
local actual
actual="$(notify_count)"
if [[ "$actual" == "$1" ]]; then
ok "$2"
else
fail "$2 -- got $actual popup(s), expected $1"
fi
}
run_watchdog() {
PATH="$WORK/bin:$PATH" \
PHONE_NUMBER="+15550000000" \
STATE_DIRECTORY="$WORK/state" \
SETTLE_SECONDS=0 \
FAILURES_BEFORE_RESTART=2 \
MAX_RESTARTS=2 \
RESTART_WINDOW=3600 \
"$WATCHDOG" 2>&1
}
expect_contains() {
if [[ "$1" == *"$2"* ]]; then
ok "$3"
else
fail "$3 -- got: $1"
fi
}
expect_missing() {
if [[ "$1" != *"$2"* ]]; then
ok "$3"
else
fail "$3 -- unexpectedly found $2"
fi
}
expect_file() {
local actual
actual="$(cat "$1" 2>/dev/null || echo MISSING)"
if [[ "$actual" == "$2" ]]; then
ok "$3"
else
fail "$3 -- $1 is '$actual', expected '$2'"
fi
}

9
signal_bot/__main__.py
View File

@ -12,10 +12,11 @@ BIND_HOST/BIND_PORT whenever they live in .env -- which is where they live.
from __future__ import annotations
import logging
import uvicorn
from signal_bot.config_env import load_config
from signal_bot.logging_setup import configure as configure_logging
from signal_bot.config import load_config
from signal_bot.portcheck import PortInUseError, assert_port_free
@ -25,7 +26,9 @@ def main() -> None:
# Before anything else: uvicorn only configures its own loggers, so
# without this the bot's own log lines go to logging.lastResort and
# anything below WARNING is thrown away.
configure_logging(config.log_level)
logging.basicConfig(
level=config.log_level, format="%(asctime)s %(levelname)s %(name)s: %(message)s"
)
try:
assert_port_free(config.bind_host, config.bind_port)
except PortInUseError as exc:

199
signal_bot/aliases.json
View File

@ -1,201 +1,18 @@
{
"_comment": "Command trigger words, including the styled Unicode variants people actually type. These live as data, not source, because they are data: adding an alias should never mean touching Python. One trap worth knowing before you edit: !poems sits two edits from !pies and passes test_no_two_commands_are_within_reach_of_each_other only because !poem is one edit away and wins the tie. Removing !poem would fail the suite at !poems, which is nowhere near where the change was made.",
"_comment": "Command trigger words, including the styled Unicode variants people actually type. These live as data, not source, because they are data: adding an alias should never mean touching Python.",
"cat": [
"!kot",
"!koty",
"!kots",
"!cat",
"!cats",
"!meow",
"!miau",
"!ᴋᴏᴛ",
"!𝓴𝓸𝓽",
"!𝗸𝗼𝘁"
"!kot", "!koty", "!kots", "!cat", "!cats", "!meow", "!miau",
"!ᴋᴏᴛ", "!𝓴𝓸𝓽", "!𝗸𝗼𝘁"
],
"dog": [
"!pies",
"!psy",
"!dog",
"!dogs",
"!woof",
"!szczek",
"!𝗽𝗶𝗲𝘀",
"!͓̽p͓̽i͓̽e͓̽s͓̽"
"!pies", "!psy", "!dog", "!dogs", "!woof", "!szczek",
"!𝗽𝗶𝗲𝘀", "!͓̽p͓̽i͓̽e͓̽s͓̽"
],
"gecko": [
"!gekon",
"!gekony",
"!geko",
"!gecko",
"!geckon",
"!geckos"
],
"trap": [
"!pułapka",
"!pulapka",
"!pułapki",
"!pulapki",
"!pułapkę",
"!pulapke",
"!trap",
"!traps",
"!trapy",
"!trapper",
"!sidła",
"!sidla",
"!sidło",
"!sidlo",
"!wnyk",
"!wnyki",
"!potrzask",
"!zasadzka",
"!klatka",
"!samołówka",
"!samolowka",
"!beartrap",
"!bear_trap",
"!snare",
"!snares",
"!mousetrap"
],
"art": [
"!sztuka",
"!sztukę",
"!sztuki",
"!art"
"!gekon", "!gekony", "!geko", "!gecko", "!geckon", "!geckos"
],
"pope": [
"!papież",
"!papiez",
"!papieze",
"!papieża",
"!papa",
"!pope",
"!popes",
"!pontifex",
"!𝗽𝗮𝗽𝗶𝗲𝘇"
],
"poem": [
"!wiersz",
"!wiersze",
"!wiersza",
"!wierszyk",
"!poem",
"!poems",
"!poezja",
"!poezje",
"!poetry"
],
"stats_today": [
"!staty",
"!statystyki",
"!stats",
"!dziś",
"!dzis"
],
"stats_yesterday": [
"!wczoraj",
"!yesterday"
],
"stats_week": [
"!tydzień",
"!tydzien",
"!week"
],
"stats_month": [
"!miesiąc",
"!miesiac",
"!month"
],
"stats_year": [
"!rok",
"!year"
],
"stats_records": [
"!rekordy",
"!rekord",
"!records"
],
"stats_crown": [
"!korona",
"!crown"
],
"stats_weekdays": [
"!dni",
"!days"
],
"stats_me": [
"!ja",
"!moje",
"!mnie"
],
"command_stats": [
"!analityka",
"!command_stats",
"!commandstats",
"!analytics",
"!uzycie",
"!użycie",
"!ranking"
],
"help": [
"!pomoc",
"!pomocy",
"!help",
"!komendy",
"!commands"
],
"add_photo": [
"!dodaj",
"!dodaj_zdjecie",
"!dodajzdjecie",
"!add_photo",
"!addphoto"
],
"random": [
"!losowe",
"!losuj",
"!random",
"!rand"
],
"suggest": [
"!zaproponuj",
"!propozycja",
"!sugestia",
"!suggest",
"!pomysl"
],
"good_bot": [
"!good bot",
"!dobry bot",
"!goodbot",
"!dobrybot",
"!good_bot",
"!dobry_bot"
],
"food_today": [
"!co_jadł",
"!co_jadl",
"!cojadł",
"!cojadl",
"!jedzenie",
"!dieta"
],
"workout_today": [
"!co_ćwiczył",
"!co_cwiczyl",
"!coćwiczył",
"!cocwiczyl",
"!trening",
"!workout"
],
"mute": [
"!cisza",
"!cicho",
"!mute"
],
"unmute": [
"!odcisz",
"!wracaj"
"!papież", "!papiez", "!papieze", "!papieża", "!papa",
"!pope", "!popes", "!pontifex", "!𝗽𝗮𝗽𝗶𝗲𝘇"
]
}

10
signal_bot/analytics/__init__.py
View File

@ -1,10 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""The command history: what was run, by whom, when, and whether it worked.
A sibling of ``stats`` rather than part of it, because the two answer
different questions from different tables. ``stats`` is the *message*
history -- who said how much -- and is what the nightly report reads.
This is the *command* history, and it exists because none of it was kept:
the old record was two aggregates written only when a command succeeded,
so a command that failed every time looked exactly like one nobody ran.
"""

146
signal_bot/analytics/adoption.py
View File

@ -1,146 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family E, the adoption half: who came back, and what a release did.
Three questions that are about the bot rather than about any one command.
* **Retention** -- of the people who used it on their first day, how many
were still using it N days later. On a history this short this is one or
two points, and the panel says so rather than drawing a curve.
* **Novelty** -- what a release actually did. ``DOCS-changelog.md`` is the
only record of when each version shipped and it is the one the maintainer
actually keeps, so the dates come from there rather than from git tags.
* **Share of chat** -- what fraction of everything said in the group was a
command. This is the only query in the package that reads
``message_events``, and it is the number that says whether the bot is a
fixture or a curiosity.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from signal_bot.stats.periods import shift
if TYPE_CHECKING:
import sqlite3
from collections.abc import Sequence
from signal_bot.analytics.scope import Scope
FIRST_DAY_SQL: Final = """
SELECT user_uuid, MIN(day) AS first_day
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'ok'
GROUP BY user_uuid
"""
ACTIVE_DAYS_SQL: Final = """
SELECT DISTINCT user_uuid, day
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'ok'
"""
SHARE_SQL: Final = """
SELECT COUNT(*) AS messages, SUM(is_command) AS commands
FROM message_events
WHERE (:group IS NULL OR group_id = :group)
AND local_hour >= 0
"""
NEW_PEOPLE_SQL: Final = """
SELECT day, COUNT(*) AS people
FROM (
SELECT user_uuid, MIN(day) AS day
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'ok'
GROUP BY user_uuid
)
GROUP BY day
ORDER BY day
"""
@dataclass(frozen=True, slots=True)
class Share:
"""How much of the group's talking is people talking to the bot."""
messages: int = 0
commands: int = 0
@property
def fraction(self) -> float:
"""Commands as a share of every counted message."""
return self.commands / self.messages if self.messages else 0.0
def retention(conn: sqlite3.Connection, scope: Scope, horizon: int) -> list[tuple[int, int, int]]:
"""(day offset, still active, cohort size) for offsets 0..horizon.
Days are stepped as dates, never as 86_400_000 milliseconds: Warsaw days
are not all the same length and the arithmetic version drifts an hour
twice a year, which would move somebody between buckets.
"""
firsts = {
str(row["user_uuid"]): str(row["first_day"])
for row in conn.execute(FIRST_DAY_SQL, scope.params())
}
if not firsts:
return []
active: set[tuple[str, str]] = {
(str(row["user_uuid"]), str(row["day"]))
for row in conn.execute(ACTIVE_DAYS_SQL, scope.params())
}
cohort = len(firsts)
return [
(
offset,
sum(1 for uuid, first in firsts.items() if (uuid, shift(first, offset)) in active),
cohort,
)
for offset in range(horizon + 1)
]
def newcomers(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, int]]:
"""(day, how many people used the bot for the first time)."""
return [
(str(row["day"]), int(row["people"]))
for row in conn.execute(NEW_PEOPLE_SQL, scope.params())
]
def share_of_chat(conn: sqlite3.Connection, scope: Scope) -> Share:
"""What fraction of the group's messages were commands.
Excludes the synthetic rows seeded from the pre-0.1.0 counter, which
carry UNKNOWN_HOUR and no is_command flag -- counting them would put a
whole day's messages in the denominator against no commands at all.
"""
# An aggregate SELECT returns one row even against an empty table, with
# NULLs in it -- so there is no None to guard against, only the NULLs.
row = conn.execute(SHARE_SQL, scope.params()).fetchone()
return Share(messages=int(row["messages"] or 0), commands=int(row["commands"] or 0))
def around_releases(
series: Sequence[tuple[str, int]], releases: Sequence[tuple[str, str]], span: int
) -> list[tuple[str, str, int, int]]:
"""(version, day, uses just after, uses just before) for each release.
"Just after" is the release day and the ``span`` days following it;
"just before" is the same number of days ending the day before. Equal
windows, so the comparison means something even at the edges of a short
history -- a release on the first recorded day gets a before-window of
zero and reads as such rather than as a spike.
"""
counts = dict(series)
out: list[tuple[str, str, int, int]] = []
for version, day in releases:
after = sum(counts.get(shift(day, offset), 0) for offset in range(span + 1))
before = sum(counts.get(shift(day, -offset), 0) for offset in range(1, span + 1))
out.append((version, day, after, before))
return out

182
signal_bot/analytics/backfill.py
View File

@ -1,182 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Recovering the command history that predates the table holding it.
Two sources, and they overlap, which is the whole difficulty. Summing them
would roughly double every historical count and nothing would fail.
* **journald** knows *which* command ran and *when*, to the second, but has
no group and no uuid -- ``dispatcher.py`` logs a display name because the
line was written to be read by a human.
* **daily_uses** knows the group, the uuid, the command and the day, and
counts them correctly, but has no times of day at all.
So the join runs over ``message_events``, which already holds the group, the
uuid, the real hour and the display name of every counted command -- and
lacks only the command's *name*, which is exactly what journald has. Each
matched row is a whole truthful record. ``daily_uses`` then fills the
**residual**: for each (group, user, command, day), whatever its count says
happened beyond what the join could account for. Nothing is counted twice.
Insert-only, always. There is no delete-then-rebuild pass, because journald
has no retention configuration and will rotate: a second run after the log
has aged out would delete rows it could no longer replace. Re-runnability
comes from the natural key and ``ON CONFLICT DO NOTHING``.
One inaccuracy, stated rather than hidden. ``dispatcher.py`` logs "running
command X" *before* the handler runs, so a journald row proves the command
was entered, not that it finished. A handler that then crashed, or a
``daily_limit`` or ``vote_gated`` refusal -- neither of which logged
anything -- is imported as ``ok`` because nothing in the record says
otherwise. Measured against the live database on 2026-08-31 the gap is about
4%: 293 reconstructed invocations against 388 successes in ``daily_uses``,
overshooting by roughly fifteen rows across all commands. Every such row
carries ``source = 'journald'``, so the UI can and does say where it came
from; only the ``live`` rows have outcomes that were actually observed.
"""
from __future__ import annotations
import datetime as dt
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from signal_bot.analytics.backfill_join import Tally, from_journal
from signal_bot.analytics.uses import (
OUTCOME_OK,
SOURCE_DAILY,
SOURCE_LIVE,
CommandEvent,
)
from signal_bot.stats.backfill import UNKNOWN_HOUR
from signal_bot.stats.periods import MS_PER_SECOND, WARSAW, weekday_of
if TYPE_CHECKING:
from collections.abc import Iterable
from signal_bot.analytics.journal import JournalLine
from signal_bot.store import Store
# Recorded per group once a pass has run, in the same place and for the same
# reason stats/backfill.py records its own marker.
BACKFILL_KEY: Final = "command_backfill"
@dataclass(frozen=True, slots=True)
class Report:
"""What one pass did, and what it refused to guess at."""
journal_rows: int = 0
daily_rows: int = 0
unmatched: int = 0
ambiguous: int = 0
groups: int = 0
def lines(self) -> tuple[str, ...]:
"""The summary a human reads. Never names anybody."""
return (
f"journald rows: {self.journal_rows}",
f"daily_uses rows: {self.daily_rows}",
f"dropped, no match: {self.unmatched}",
f"dropped, ambiguous: {self.ambiguous}",
f"groups marked: {self.groups}",
)
def midnight_ms(day: str) -> int:
"""The first millisecond of a Warsaw calendar day.
Built from the date rather than by subtracting 86_400_000 from anything:
Warsaw days are not all 24 hours, and the arithmetic spelling lands an
hour out twice a year.
"""
start = dt.datetime.combine(dt.date.fromisoformat(day), dt.time.min, tzinfo=WARSAW)
return int(start.timestamp() * MS_PER_SECOND)
def live_floor(store: Store) -> int:
"""The earliest live row, or "no live rows yet".
Every backfilled row must be strictly older than this. Without the
cutoff, a second pass run after live recording had started would import
journald lines describing invocations that already exist as ``live``
rows -- with a different source, so the natural key would not stop them.
"""
row = store.commands.connection.execute(
"SELECT MIN(invoked_at_ms) AS floor FROM command_events WHERE source = ?",
(SOURCE_LIVE,),
).fetchone()
floor = row["floor"]
# No live rows yet means nothing is off limits.
return int(floor) if floor is not None else 1 << 62
def _residual(store: Store, tally: Tally) -> list[CommandEvent]:
"""Fill in whatever daily_uses counted that the join could not explain.
``uses`` is the truth about how many times something ran that day. The
join accounts for some of them exactly; this adds the remainder at day
granularity, with no hour and no durations, so the total is right even
where the detail is missing.
"""
events: list[CommandEvent] = []
rows = store.commands.connection.execute(
"SELECT group_id, user_uuid, command, day, uses FROM daily_uses ORDER BY day, command"
).fetchall()
for row in rows:
key = (str(row["group_id"]), str(row["user_uuid"]), str(row["command"]), str(row["day"]))
remaining = int(row["uses"]) - tally.matched[key]
day = str(row["day"])
base = midnight_ms(day)
events.extend(
CommandEvent(
group_id=key[0],
user_uuid=key[1],
command=key[2],
outcome=OUTCOME_OK,
# Deterministic, so a second pass produces the identical key
# and inserts nothing. The same trick, for the same reason,
# as seed_from_counter's sent_at_ms=index + 1.
invoked_at_ms=base + index,
day=day,
local_hour=UNKNOWN_HOUR,
weekday=weekday_of(day),
source=SOURCE_DAILY,
)
for index in range(max(0, remaining))
)
return events
def plan(store: Store, lines: Iterable[JournalLine]) -> tuple[Report, list[CommandEvent]]:
"""Work out what would be imported, writing nothing.
Separate from ``apply`` so that ``--dry-run`` reports the real numbers
rather than an estimate of them. Rolling back would not have worked:
``CommandEventStore.record`` commits per row, exactly as ``EventStore``
does, so by the time a caller could roll back there is nothing left to
roll back.
"""
floor = live_floor(store)
tally = from_journal(store, lines, floor)
residual = _residual(store, tally)
events = [*tally.events, *residual]
return (
Report(
journal_rows=len(tally.events),
daily_rows=len(residual),
unmatched=tally.unmatched,
ambiguous=tally.ambiguous,
groups=len({event.group_id for event in events}),
),
events,
)
def apply(store: Store, events: Iterable[CommandEvent]) -> None:
"""Write a planned import. Insert-only; a repeat run inserts nothing."""
groups = set()
for event in events:
store.commands.record(event)
groups.add(event.group_id)
for group_id in sorted(groups):
store.set_meta(group_id, BACKFILL_KEY, "done")

180
signal_bot/analytics/backfill_join.py
View File

@ -1,180 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Attaching a command name from the journal to a message that was one.
Split out of ``backfill.py`` so both stay under the 250-line cap. This half
is the part that can be wrong, so it is the part that refuses to guess.
The direction matters and is easy to get backwards. Iterating the journal
and looking for a message means resolving a display name to a uuid and a
group, which the journal does not know. Iterating ``message_events`` instead
means every row already carries the group, the uuid, the day and the real
hour, and the journal supplies the one field missing -- the command's name.
"""
from __future__ import annotations
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Final
from signal_bot.analytics.uses import (
OUTCOME_COOLDOWN,
OUTCOME_OK,
SOURCE_JOURNAL,
CommandEvent,
)
from signal_bot.stats.periods import weekday_of
if TYPE_CHECKING:
import sqlite3
from collections.abc import Iterable, Sequence
from signal_bot.analytics.journal import JournalLine
from signal_bot.store import Store
# How far apart a message and the log line about it may be and still be the
# same event.
#
# SYMMETRIC, and that is not obvious. The log line is written strictly after
# the message arrives, so an asymmetric [0, +W] window looks correct -- and
# throws away most of the data. Measured against the live database on
# 2026-08-31: of 299 counted commands, 194 have a journal line *earlier*
# than their own sent_at_ms, by up to 3.2 seconds, median -173 ms.
#
# The reason is that the two stamps come from two different clocks.
# ``sent_at_ms`` is Signal's envelope timestamp, set on the *sender's phone*;
# the journal stamp is this host's. Nothing keeps those in step, so their
# difference is skew and can have either sign.
#
# 15 seconds each way is far wider than the 3.2 s of skew actually seen and
# still narrower than a person's own repeat rate. Widening it cannot corrupt
# anything either way: two different commands inside one window are dropped
# as ambiguous rather than guessed at.
WINDOW_MS: Final = 15_000
@dataclass(slots=True)
class _Candidate:
"""One journal line, and whether the join has already spent it."""
line: JournalLine
used: bool = False
@dataclass(slots=True)
class Tally:
"""The running result of a pass."""
events: list[CommandEvent] = field(default_factory=list)
matched: Counter[tuple[str, str, str, str]] = field(default_factory=Counter)
unmatched: int = 0
ambiguous: int = 0
def _by_name(lines: Iterable[JournalLine]) -> dict[str, list[_Candidate]]:
"""Group journal lines by the display name they were logged for."""
grouped: dict[str, list[_Candidate]] = defaultdict(list)
for line in lines:
grouped[line.display_name].append(_Candidate(line))
for candidates in grouped.values():
candidates.sort(key=lambda candidate: candidate.line.at_ms)
return dict(grouped)
def _assign(
rows: Sequence[sqlite3.Row], grouped: dict[str, list[_Candidate]]
) -> dict[int, _Candidate]:
"""Pair each counted command with the journal line most likely to be it.
Closest pair first, globally, rather than best-for-each-row in turn.
That difference is what makes two commands from one person seconds apart
resolvable at all.
Per-row greedy sees both lines inside the window, cannot tell which is
which, and drops both. Global assignment can: the clock skew between a
sender's phone and this host is *systematic*, so the true pairs share
almost the same offset and are the two smallest distances available.
Ties break on row order and then line order, so a re-run against the
same inputs produces the same pairing. The backfill has to be
re-runnable, and "mostly the same" is not that.
"""
pairs: list[tuple[int, int, int]] = []
for index, row in enumerate(rows):
candidates = grouped.get(str(row["display_name"]), [])
for position, candidate in enumerate(candidates):
delta = candidate.line.at_ms - int(row["sent_at_ms"])
if abs(delta) <= WINDOW_MS:
pairs.append((abs(delta), index, position))
pairs.sort()
taken: dict[int, _Candidate] = {}
for _distance, index, position in pairs:
if index in taken:
continue
candidate = grouped[str(rows[index]["display_name"])][position]
if candidate.used:
continue
candidate.used = True
taken[index] = candidate
return taken
def _in_window(row: sqlite3.Row, grouped: dict[str, list[_Candidate]]) -> bool:
"""Whether the journal held any line at all for this message."""
sent_at_ms = int(row["sent_at_ms"])
return any(
abs(candidate.line.at_ms - sent_at_ms) <= WINDOW_MS
for candidate in grouped.get(str(row["display_name"]), [])
)
def from_journal(store: Store, lines: Iterable[JournalLine], floor: int) -> Tally:
"""Walk the counted commands and attach a name to each from the journal."""
tally = Tally()
grouped = _by_name(lines)
rows = store.events.connection.execute(
"""
SELECT group_id, uuid, display_name, sent_at_ms, day, local_hour
FROM message_events
WHERE is_command = 1 AND local_hour >= 0 AND sent_at_ms < ?
ORDER BY sent_at_ms
""",
(floor,),
).fetchall()
taken = _assign(rows, grouped)
for index, row in enumerate(rows):
chosen = taken.get(index)
if chosen is None:
# A message the journal cannot explain. Either it never held a
# line for it -- vacuumed, or older than the log -- or every line
# in its window went to a closer message, which is the honest
# reading of "two of these happened and I have one record".
# Dropped either way: a wrong attribution is worse than a gap,
# because it is indistinguishable from data.
if _in_window(row, grouped):
tally.ambiguous += 1
else:
tally.unmatched += 1
continue
line = chosen.line
outcome = OUTCOME_COOLDOWN if line.refused else OUTCOME_OK
day = str(row["day"])
tally.events.append(
CommandEvent(
group_id=str(row["group_id"]),
user_uuid=str(row["uuid"]),
command=line.command,
outcome=outcome,
invoked_at_ms=int(row["sent_at_ms"]),
day=day,
local_hour=int(row["local_hour"]),
weekday=weekday_of(day),
source=SOURCE_JOURNAL,
)
)
if outcome == OUTCOME_OK:
# Only successes are in daily_uses, so only successes may be
# subtracted from it below.
tally.matched[(str(row["group_id"]), str(row["uuid"]), line.command, day)] += 1
return tally

106
signal_bot/analytics/content.py
View File

@ -1,106 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family F: which items get served, and how often the same one comes back.
Reads ``item_id``, which holds public identifiers only -- a Wikidata QID, a
Commons page id, a Wolne Lektury slug, a content-addressed filename. Never a
title, a caption or a name. That constraint is what makes this family
possible at all: "the pope pool feels repetitive" is an impression, and
"106 draws produced 61 distinct popes" is a number.
The repeat rate is the headline. For a pool of size N drawn uniformly, the
expected number of distinct items after k draws is N(1 - (1 - 1/N)^k), so a
distinct count well below that means the draw is not uniform -- which is
exactly the bug that put one identical photograph into a live group chat
when iNaturalist silently ignored ``order_by=random``.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
import sqlite3
from signal_bot.analytics.scope import Scope
ITEMS_SQL: Final = """
SELECT command,
COUNT(*) AS draws,
COUNT(DISTINCT item_id) AS distinct_items
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND item_id IS NOT NULL
AND outcome = 'ok'
GROUP BY command
ORDER BY draws DESC, command
"""
TOP_ITEMS_SQL: Final = """
SELECT command, item_id, COUNT(*) AS draws
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND item_id IS NOT NULL
AND outcome = 'ok'
GROUP BY command, item_id
HAVING draws > 1
ORDER BY draws DESC, command, item_id
"""
@dataclass(frozen=True, slots=True)
class Pool:
"""One command's draws, and how varied they turned out to be."""
command: str
draws: int
distinct: int
@property
def repeat_rate(self) -> float:
"""Share of draws that produced something already seen.
Zero means every draw was new. Approaching one means the pool is
exhausted -- or was never as large as it looked.
"""
if self.draws <= 0:
return 0.0
return 1.0 - (self.distinct / self.draws)
def pools(conn: sqlite3.Connection, scope: Scope) -> list[Pool]:
"""Every command that names what it served, most drawn first."""
return [
Pool(
command=str(row["command"]),
draws=int(row["draws"]),
distinct=int(row["distinct_items"]),
)
for row in conn.execute(ITEMS_SQL, scope.params())
]
def repeats(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, str, int]]:
"""(command, item, times) for every item served more than once.
The identifiers are safe to show -- that is the whole reason the column
may exist -- but they are only meaningful next to the command, so they
are never returned on their own.
"""
return [
(str(row["command"]), str(row["item_id"]), int(row["draws"]))
for row in conn.execute(TOP_ITEMS_SQL, scope.params())
]
def expected_distinct(pool_size: int, draws: int) -> float:
"""How many distinct items a *uniform* draw would have produced.
The comparison that turns a repeat rate into a verdict. A pool of 266
traps drawn 19 times should yield about 18.4 distinct ones; yielding 9
would mean the randomness is not what it claims, which has happened in
this repository before and reached a live group chat.
"""
if pool_size <= 0 or draws <= 0:
return 0.0
return pool_size * (1.0 - ((pool_size - 1) / pool_size) ** draws)

102
signal_bot/analytics/demand.py
View File

@ -1,102 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family D: what the group asked for that does not exist.
The management page's Ideas tab already lists ``unknown_commands`` with the
words and the messages they came in. This adds the part it does not have:
the same words **grouped by the trigger they nearly hit**, so that "17
near-misses on !pułapka" reads as *this command wants another alias* rather
than as seventeen unrelated typos.
The grouping is done here rather than in SQL because the distance function
is ``nearest``'s, and it is optimal string alignment rather than a ratio --
a transposition costs one edit there and shreds the matching blocks a ratio
counts. Reimplementing that in SQL is not possible and approximating it
would put a different answer on the page from the one the bot gives in the
group.
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from signal_bot.nearest import nearest
if TYPE_CHECKING:
import sqlite3
from collections.abc import Sequence
from signal_bot.analytics.scope import Scope
WORDS_SQL: Final = """
SELECT word, COUNT(*) AS hits, MAX(sent_at_ms) AS last_at
FROM unknown_commands
WHERE (:group IS NULL OR group_id = :group)
GROUP BY word
ORDER BY hits DESC, word
"""
FILED_SQL: Final = """
SELECT COUNT(*) AS filed
FROM suggestions
WHERE (:group IS NULL OR group_id = :group)
"""
@dataclass(frozen=True, slots=True)
class NearMiss:
"""A real trigger, and how often somebody nearly typed it."""
target: str
hits: int
variants: int
@dataclass(frozen=True, slots=True)
class Demand:
"""What the group tried to type, and what it filed on purpose."""
near_misses: tuple[NearMiss, ...] = ()
unclaimed: int = 0
unclaimed_words: int = 0
filed: int = 0
def collect(conn: sqlite3.Connection, scope: Scope, triggers: Sequence[str]) -> Demand:
"""Group every unrecognised word by the command it was probably meant to be.
A word that resolves to nothing at all is counted but not attributed:
those are the genuinely new ideas, and pretending one of them is a typo
of an existing command is how a real request gets filed as noise.
The words themselves never leave this function. A half-typed name is
still a name -- ``check_personal_data`` learns the names it refuses from
the live database, so it catches ``anna`` and could never catch
``!annna`` -- and the Ideas tab is the one page that already carries
them, deliberately, and never writes them into the repository.
"""
hits: Counter[str] = Counter()
variants: Counter[str] = Counter()
unclaimed = 0
unclaimed_words = 0
for row in conn.execute(WORDS_SQL, scope.params()):
word = str(row["word"])
count = int(row["hits"])
target = nearest(word, triggers)
if target is None:
unclaimed += count
unclaimed_words += 1
continue
hits[target] += count
variants[target] += 1
filed_row = conn.execute(FILED_SQL, scope.params()).fetchone()
return Demand(
near_misses=tuple(
NearMiss(target=target, hits=count, variants=variants[target])
for target, count in sorted(hits.items(), key=lambda pair: (-pair[1], pair[0]))
),
unclaimed=unclaimed,
unclaimed_words=unclaimed_words,
filed=int(filed_row["filed"] or 0) if filed_row is not None else 0,
)

194
signal_bot/analytics/health.py
View File

@ -1,194 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family C: which commands are broken, slow, or refusing people.
This is the family that had *no* data at all before ``command_events``:
``store.record_use`` fires only on success, so a command that crashed every
time was indistinguishable from one nobody ran.
Latency narrows to ``live``. A reconstructed row has no duration -- NULL,
never 0, because a zero would drag a p50 down and read as a fast command.
Every aggregate here therefore says ``handler_ms IS NOT NULL`` rather than
relying on the source column, so a future third source with real timings
would be included automatically and one without would not.
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
import sqlite3
from signal_bot.analytics.scope import Scope
# Which percentiles the panel shows. The median says what it is usually
# like; the 95th says what it is like when somebody complains.
P50: Final = 0.50
P95: Final = 0.95
@dataclass(frozen=True, slots=True)
class Latency:
"""How long one command takes, split the way DOCS-latency.md splits it."""
command: str
samples: int
handler_p50: int
handler_p95: int
send_p50: int
upstream_p50: int
@dataclass(frozen=True, slots=True)
class Upstream:
"""One external service, and how well it has been behaving."""
host: str
calls: int
failures: int
p50_ms: int
@property
def failure_rate(self) -> float:
"""Share of calls that were not a 2xx."""
return self.failures / self.calls if self.calls else 0.0
GATES_SQL: Final = """
SELECT command, outcome, COUNT(*) AS n
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome LIKE 'refused_%'
GROUP BY command, outcome
ORDER BY n DESC, command
"""
ERRORS_SQL: Final = """
SELECT command, error_class, COUNT(*) AS n
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'error'
GROUP BY command, error_class
ORDER BY n DESC, command
"""
TIMED_SQL: Final = """
SELECT command, handler_ms, send_ms
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND handler_ms IS NOT NULL
ORDER BY command, handler_ms
"""
UPSTREAM_SQL: Final = """
SELECT host,
COUNT(*) AS calls,
SUM(status < 200 OR status > 299) AS failures
FROM upstream_calls
WHERE (:group IS NULL OR group_id = :group)
GROUP BY host
ORDER BY calls DESC, host
"""
UPSTREAM_TIMES_SQL: Final = """
SELECT host, duration_ms
FROM upstream_calls
WHERE (:group IS NULL OR group_id = :group)
ORDER BY host, duration_ms
"""
COMMAND_UPSTREAM_SQL: Final = """
SELECT command, duration_ms
FROM upstream_calls
WHERE (:group IS NULL OR group_id = :group)
ORDER BY command, duration_ms
"""
def percentile(sorted_values: list[int], fraction: float) -> int:
"""The value at a fraction of the way through an already-sorted list.
Nearest-rank: ``ceil(p * n) - 1``, so the answer is always one of the
measurements rather than an interpolation between two of them. A p95
that nobody actually experienced is a worse number to act on than a
real one that is slightly off the theoretical quantile.
Spelled with ``math.ceil`` rather than ``round`` deliberately. Python
rounds halves to even, so ``round(0.5)`` is 0 and ``round(1.5)`` is 2 --
which would make the median of a two-element sample depend on the
sample's length in a way no reader would predict.
Computed here rather than in SQL: SQLite has no percentile function
without an extension, and adding one would mean a seventh dependency for
three lines of arithmetic.
"""
if not sorted_values:
return 0
rank = math.ceil(fraction * len(sorted_values)) - 1
return sorted_values[max(0, min(len(sorted_values) - 1, rank))]
def latencies(conn: sqlite3.Connection, scope: Scope) -> list[Latency]:
"""Per-command timings, from the rows that actually carry them."""
handler: dict[str, list[int]] = {}
send: dict[str, list[int]] = {}
for row in conn.execute(TIMED_SQL, scope.params()):
command = str(row["command"])
handler.setdefault(command, []).append(int(row["handler_ms"]))
if row["send_ms"] is not None:
send.setdefault(command, []).append(int(row["send_ms"]))
upstream: dict[str, list[int]] = {}
for row in conn.execute(COMMAND_UPSTREAM_SQL, scope.params()):
upstream.setdefault(str(row["command"]), []).append(int(row["duration_ms"]))
for values in (*handler.values(), *send.values(), *upstream.values()):
values.sort()
return sorted(
(
Latency(
command=command,
samples=len(values),
handler_p50=percentile(values, P50),
handler_p95=percentile(values, P95),
send_p50=percentile(send.get(command, []), P50),
upstream_p50=percentile(upstream.get(command, []), P50),
)
for command, values in handler.items()
),
key=lambda entry: (-entry.handler_p50, entry.command),
)
def gates(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, str, int]]:
"""(command, which gate, how often). A high count is mis-tuning."""
return [
(str(row["command"]), str(row["outcome"]), int(row["n"]))
for row in conn.execute(GATES_SQL, scope.params())
]
def errors(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, str, int]]:
"""(command, exception class, how often). Never an exception message."""
return [
(str(row["command"]), str(row["error_class"]), int(row["n"]))
for row in conn.execute(ERRORS_SQL, scope.params())
]
def upstreams(conn: sqlite3.Connection, scope: Scope) -> list[Upstream]:
"""Every external service the bot depends on, and how it is holding up."""
times: dict[str, list[int]] = {}
for row in conn.execute(UPSTREAM_TIMES_SQL, scope.params()):
times.setdefault(str(row["host"]), []).append(int(row["duration_ms"]))
for values in times.values():
values.sort()
return [
Upstream(
host=str(row["host"]),
calls=int(row["calls"]),
failures=int(row["failures"] or 0),
p50_ms=percentile(times.get(str(row["host"]), []), P50),
)
for row in conn.execute(UPSTREAM_SQL, scope.params())
]

149
signal_bot/analytics/hours.py
View File

@ -1,149 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family B, the time-of-day half: when the bot is actually used.
Every query here filters ``local_hour >= 0``. That is the whole point of the
filter: ``daily_uses`` rows carry ``UNKNOWN_HOUR`` because their source knew
only a date, and letting them through would file a third of the history
under hour -1 -- or, worse, under midnight if anybody later "tidied" the
sentinel to 0.
``journald`` rows *are* included. They were matched to a real message, so
their hour was observed even though their duration was not. Excluding them
would throw away most of the history for no gain; that distinction is why
``scope`` names two predicates rather than one.
A Warsaw day is not always 24 hours -- 2026-03-29 has 23 and 2026-10-25 has
25 -- so callers must not assume the buckets sum to a day, and nothing here
divides by 24.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
import sqlite3
from signal_bot.analytics.scope import Scope
HOURS_IN_DAY: Final = 24
DAYS_IN_WEEK: Final = 7
@dataclass(frozen=True, slots=True)
class Peak:
"""The busiest and quietest hours, when there is enough to say."""
busiest: int = -1
busiest_uses: int = 0
quietest: int = -1
quietest_uses: int = 0
BY_HOUR_SQL: Final = """
SELECT local_hour AS hour, COUNT(*) AS uses
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND local_hour >= 0
AND outcome != 'unknown_word'
GROUP BY local_hour
ORDER BY local_hour
"""
BY_WEEKDAY_HOUR_SQL: Final = """
SELECT weekday, local_hour AS hour, COUNT(*) AS uses
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND local_hour >= 0
AND weekday >= 0
AND outcome != 'unknown_word'
GROUP BY weekday, local_hour
ORDER BY weekday, local_hour
"""
BY_COMMAND_HOUR_SQL: Final = """
SELECT command, local_hour AS hour, COUNT(*) AS uses
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND local_hour >= 0
AND outcome != 'unknown_word'
GROUP BY command, local_hour
ORDER BY command, local_hour
"""
BY_WEEKDAY_SQL: Final = """
SELECT weekday, COUNT(*) AS uses, COUNT(DISTINCT day) AS days
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND weekday >= 0
AND outcome != 'unknown_word'
GROUP BY weekday
ORDER BY weekday
"""
def by_hour(conn: sqlite3.Connection, scope: Scope) -> list[int]:
"""Uses per hour of the day, as 24 buckets with the empty ones present.
A dense list rather than the rows sqlite returns: a bar chart with a
missing 04:00 draws the wrong shape, and "nobody uses it at four in the
morning" is one of the things the panel is for.
"""
# Indexed without a bounds check: local_hour is written by hour_of(),
# which cannot return anything outside 0-23, and the query already
# excludes the UNKNOWN_HOUR sentinel. A guard here would be dead code
# that hides a corrupt row instead of failing on one.
buckets = [0] * HOURS_IN_DAY
for row in conn.execute(BY_HOUR_SQL, scope.params()):
buckets[int(row["hour"])] = int(row["uses"])
return buckets
def by_weekday_hour(conn: sqlite3.Connection, scope: Scope) -> list[list[int]]:
"""A 7x24 grid, Monday first, matching date.weekday()."""
grid = [[0] * HOURS_IN_DAY for _ in range(DAYS_IN_WEEK)]
for row in conn.execute(BY_WEEKDAY_HOUR_SQL, scope.params()):
grid[int(row["weekday"])][int(row["hour"])] = int(row["uses"])
return grid
def by_command_hour(conn: sqlite3.Connection, scope: Scope) -> dict[str, list[int]]:
"""One 24-bucket row per command: which commands belong to which hours."""
grid: dict[str, list[int]] = {}
for row in conn.execute(BY_COMMAND_HOUR_SQL, scope.params()):
buckets = grid.setdefault(str(row["command"]), [0] * HOURS_IN_DAY)
buckets[int(row["hour"])] = int(row["uses"])
return grid
def by_weekday(conn: sqlite3.Connection, scope: Scope) -> list[tuple[int, int, int]]:
"""(weekday, uses, days seen) so an average can be taken honestly.
Dividing by the number of weeks would be wrong on a history this short:
a Tuesday that has happened once is not a weekly average.
"""
return [
(int(row["weekday"]), int(row["uses"]), int(row["days"]))
for row in conn.execute(BY_WEEKDAY_SQL, scope.params())
]
def peak(buckets: list[int]) -> Peak:
"""The busiest hour, and the quietest hour that anybody ever used.
"Quietest" skips hours with nothing at all: at three days old most of
the night is empty, and "the deadest hour is 04:00 with zero" would be
true of about nine different hours and interesting about none of them.
"""
if not any(buckets):
return Peak()
used = [(hour, uses) for hour, uses in enumerate(buckets) if uses]
busiest = max(used, key=lambda pair: (pair[1], -pair[0]))
quietest = min(used, key=lambda pair: (pair[1], pair[0]))
return Peak(
busiest=busiest[0],
busiest_uses=busiest[1],
quietest=quietest[0],
quietest_uses=quietest[1],
)

90
signal_bot/analytics/journal.py
View File

@ -1,90 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Reading the bot's own journal back into the command history.
Pure parsing, no I/O: the lines arrive as an iterable so that the joining
logic and its tests never shell out, and so that a captured log file works
exactly as well as a live ``journalctl``.
The journal is the only record of what ran before ``command_events``
existed, and it is a poor one. It carries no group id and no uuid -- only a
*display name*, because ``dispatcher.py``'s log line was written to be read
by a human. That is why the join in ``backfill`` runs the other way round,
over ``message_events``, which knows everything except which command it was.
A display name read here is a lookup key and nothing else. It never reaches
a row, a log line or a file: ``group_people`` turns it into a uuid inside
the joiner and the string is dropped. ``DOCS-privacy.md`` names the journal
as the one place display names live, and ``--vacuum-time=`` as how that is
emptied -- so this parser must never be the thing that copies them out.
"""
from __future__ import annotations
import datetime as dt
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
MS_PER_SECOND: Final = 1000
# `journalctl -o short-iso` prefixes each line with an offset-aware timestamp
# -- 2026-08-31T21:28:26+02:00 -- which is the only spelling in the file that
# survives a DST boundary. The bot's own asctime, further along the same
# line, is naive and would be an hour wrong for half the year.
_RUNNING: Final = re.compile(r"^(?P<at>\S+)\s.*?running command (?P<command>\S+) for (?P<who>.+)$")
# The one gate that already logged its refusals. daily_limit and vote_gated
# logged nothing at all, so those simply cannot be recovered.
_REFUSED: Final = re.compile(
r"^(?P<at>\S+)\s.*?cooldown refused (?P<command>\S+) for (?P<who>.+?): told them"
)
@dataclass(frozen=True, slots=True)
class JournalLine:
"""One thing the bot said it did, with the name it said it for."""
at_ms: int
command: str
display_name: str
refused: bool
def _at_ms(stamp: str) -> int | None:
"""Parse the journal's own timestamp, or None if this is not one."""
try:
moment = dt.datetime.fromisoformat(stamp)
except ValueError:
return None
# An offset-naive stamp would silently be read as UTC and land the whole
# import two hours out, which is exactly the class of error the hour
# columns exist to avoid. Refuse it instead.
if moment.tzinfo is None:
return None
return int(moment.timestamp() * MS_PER_SECOND)
def parse(lines: Iterable[str]) -> Iterator[JournalLine]:
"""Yield one record per line the bot wrote about running a command.
Everything else -- startup banners, tracebacks, envelope shapes, other
units -- simply does not match and is skipped. The parser is deliberately
permissive about what sits between the timestamp and the phrase, because
the journal identifier is ``run.sh[<pid>]`` and the pid changes.
"""
for line in lines:
for pattern, refused in ((_RUNNING, False), (_REFUSED, True)):
found = pattern.match(line)
if found is None:
continue
at_ms = _at_ms(found["at"])
if at_ms is not None:
yield JournalLine(
at_ms=at_ms,
command=found["command"],
display_name=found["who"].strip(),
refused=refused,
)
break

131
signal_bot/analytics/probe.py
View File

@ -1,131 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""What one running command did, collected without changing any signature.
Timing a command means knowing things the handler learns and the dispatcher
records: how long the upstream fetch took, how long the send took, which
item was served. Threading that back would mean a return value or an extra
argument on every handler, every fetcher method and every reply helper.
A ``ContextVar`` avoids all of it. The dispatcher opens a probe around the
handler call; anything running underneath finds it and notes what it did.
``contextvars`` is stdlib, so the runtime stays at exactly six dependencies.
``Bot.listen`` runs every frame in a task of its own (``inflight.py``), and
a task starts with a *copy* of the context, so frames handled side by side
cannot see each other's probe: each ``probing()`` sets the variable in its
own copy, and the reset in one task cannot be undone by another. The same
holds for a handler that spawns a task of its own -- the copy's notes land
in that handler's probe, nowhere else.
Every ``note_*`` is a no-op when nothing is probing. That branch lives here,
once, so no caller needs to ask whether it is being measured -- which is
what keeps ``http_client`` and ``context`` free of analytics conditionals.
"""
from __future__ import annotations
import contextlib
import time
from contextvars import ContextVar
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterator
# No default sentinel object: None is the whole vocabulary, and the helpers
# below are the only readers.
CURRENT: ContextVar[Probe | None] = ContextVar("command_probe", default=None)
# A request that never got an answer at all -- DNS, connect, timeout. Kept
# distinct from a 5xx, which is upstream answering and answering badly; the
# two mean different things when a command starts failing.
NO_RESPONSE: Final = 0
MS_PER_SECOND: Final = 1000
@dataclass(frozen=True, slots=True)
class UpstreamCall:
"""One outbound HTTP request made while serving a command."""
host: str
status: int
ms: int
@dataclass(slots=True)
class Probe:
"""The mutable notes one invocation accumulates.
Upstream calls are *buffered* rather than written as they happen: they
belong to a ``command_events`` row that does not exist yet, because the
outcome and the duration are only known once the handler has returned.
"""
send_ms: int = 0
item_id: str | None = None
calls: list[UpstreamCall] = field(default_factory=list)
@contextlib.contextmanager
def probing() -> Iterator[Probe]:
"""Open a probe for the duration of one command, and always close it."""
active = Probe()
token = CURRENT.set(active)
try:
yield active
finally:
CURRENT.reset(token)
def note_send(ms: int) -> None:
"""Add the cost of one outbound Signal send.
Accumulated rather than assigned: a handler may send twice, and
``!pułapka`` does -- a picture and then its description.
"""
active = CURRENT.get()
if active is not None:
active.send_ms += ms
@contextlib.asynccontextmanager
async def timed_send() -> AsyncIterator[None]:
"""Time one outbound send and note it, whether or not it succeeds.
``DOCS-latency.md`` measured ``POST /v2/send`` at 2.65 s of a 3.7 s
reply -- 72% of the wall clock, none of it this repository's code -- by
reading three log lines by hand, once. This makes that measurement
continuous, which is what the document's own rule asks for: an upstream
latency claim gets measured before it gets designed around.
"""
started = time.perf_counter()
try:
yield
finally:
note_send(round((time.perf_counter() - started) * MS_PER_SECOND))
def note_upstream(host: str, status: int, ms: int) -> None:
"""Add one upstream request. Host only -- never the URL.
A URL carries identifiers, and some of them are the ones ``item_id``
exists to govern. A hostname cannot carry anything.
"""
active = CURRENT.get()
if active is not None:
active.calls.append(UpstreamCall(host=host, status=status, ms=ms))
def note_item(item_id: str) -> None:
"""Record which item was served, as a public identifier.
A Wikidata QID, a Commons page id, a Wolne Lektury slug, a pseudonym
key, a content-addressed filename. Never a title, a caption or a name:
this is what makes "the pope pool feels repetitive" a number instead of
an impression, and an identifier cannot carry a person.
"""
active = CURRENT.get()
if active is not None:
active.item_id = item_id

196
signal_bot/analytics/recorder.py
View File

@ -1,196 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Turning one attempted command into one row of history.
This is where the dispatcher's ``try/except`` around a handler now lives, so
that the dispatcher gains a ``with`` rather than forty lines and stays under
the 250-line cap. It also means the six outcome branches can be tested
directly, without driving a whole websocket frame through to reach an
exception path.
The invariant that must survive every future edit to this file: recording a
*use* and recording an *attempt* are two different writes with two different
rules, and they may never be merged.
* ``store.record_use`` is success-only and feeds the cooldown and the daily
limit. ``CommandRefusedError``'s own docstring records what happened when
a refusal reached it: the cooldown window slid forward every time somebody
was told to wait, so waiting the stated time never helped.
* ``store.commands.record`` takes every attempt, including the refusals and
the crashes -- which is the entire point, since a command that fails every
time used to be indistinguishable from one nobody ran.
The temptation this creates is to hoist one call above the other, or to route
both through a single store method. ``test_dispatcher_recording`` fires a
refused invocation and asserts both that the history grew *and* that
``last_used`` did not move, which is the cheapest way to notice.
"""
from __future__ import annotations
import contextlib
import logging
import time
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from signal_bot.analytics.uses import (
ARGS_PRESENT,
OUTCOME_COOLDOWN,
OUTCOME_DAILY,
OUTCOME_ERROR,
OUTCOME_OFF,
OUTCOME_OK,
OUTCOME_VOTE,
CommandEvent,
)
from signal_bot.decorators import GATE_COOLDOWN, GATE_DAILY, GATE_OFF, GATE_VOTE
from signal_bot.registry import CommandRefusedError
from signal_bot.stats.periods import day_of, hour_of, weekday_of
if TYPE_CHECKING:
from collections.abc import Iterator
from signal_bot.analytics import probe
from signal_bot.analytics.uses import CommandEventStore
logger = logging.getLogger(__name__)
MS_PER_SECOND: Final = 1000
# A gate that did not name itself. Reachable only from a bare
# ``raise CommandRefusedError``, which nothing in this repository does any
# more -- but the constructor still allows it, so the mapping must too.
OUTCOME_REFUSED: Final = "refused_cooldown"
_BY_GATE: Final = {
GATE_COOLDOWN: OUTCOME_COOLDOWN,
GATE_DAILY: OUTCOME_DAILY,
GATE_VOTE: OUTCOME_VOTE,
GATE_OFF: OUTCOME_OFF,
}
def refusal_outcome(gate: str) -> str:
"""Name the outcome one gate's refusal should be filed under."""
return _BY_GATE.get(gate, OUTCOME_REFUSED)
def args_token(args: tuple[str, ...]) -> str:
"""Reduce a command's arguments to something that cannot hold a name.
The registry has already replaced a photo album's arguments with their
pseudonym keys before the dispatcher sees them -- that is what
``arg_resolver`` is for, and why it runs at resolution rather than in a
handler. Those keys are safe and worth keeping: "which person is asked
for most" is a real question.
Anything else becomes a flag. A column of words people typed is exactly
the shape that put real names in this repository once, and the personal
data gate learns the names it refuses from the live database -- so it
would catch a name typed correctly and could never catch a typo.
"""
if not args:
return ""
# One resolved token is a key. Several words are words.
if len(args) == 1 and args[0].isalnum():
return args[0]
return ARGS_PRESENT
@dataclass(slots=True)
class Attempt:
"""What the dispatcher learned by trying to run one command."""
outcome: str = OUTCOME_OK
error_class: str = ""
handler_ms: int = 0
@property
def ok(self) -> bool:
"""True when the handler actually ran to completion.
The dispatcher asks this rather than comparing to a constant, so
"did it work" has one spelling and the *use* -- the write that feeds
the cooldown -- cannot accidentally start counting refusals.
"""
return self.outcome == OUTCOME_OK
@contextlib.contextmanager
def attempting(command: str) -> Iterator[Attempt]:
"""Run one handler, and come out knowing how it went and what it cost.
Swallows the handler's exception rather than re-raising: the dispatcher
has always treated a failed command as an answered frame, because a raise
here ends the receive loop and the bot goes silently deaf until the
watchdog notices. That behaviour is unchanged -- only now the failure
leaves a row behind.
"""
attempt = Attempt()
started = time.perf_counter()
try:
yield attempt
except CommandRefusedError as refusal:
attempt.outcome = refusal_outcome(refusal.gate)
logger.debug("command %s refused by a gate", command)
except Exception as error:
# The class name, never str(error): an exception's message carries
# whatever it was processing, and here that is somebody's message.
# "17 ImageFetchError" is the whole diagnostic value anyway.
attempt.outcome = OUTCOME_ERROR
attempt.error_class = type(error).__name__
logger.exception("command %s failed", command)
attempt.handler_ms = round((time.perf_counter() - started) * MS_PER_SECOND)
@dataclass(frozen=True, slots=True)
class Invocation:
"""Who ran what, where, and when the message that asked was sent.
Grouped rather than passed as five arguments, because these five always
travel together and none of them means anything without the rest.
"""
group_id: str
user_uuid: str
command: str
args: tuple[str, ...]
sent_at_ms: int
def record_attempt(
store: CommandEventStore,
invocation: Invocation,
attempt: Attempt,
notes: probe.Probe,
) -> None:
"""Append one attempt, and never let doing so break the bot.
The whole body is guarded for the reason ``_answer_unknown``'s is: this
runs inside the receive loop, and an exception escaping it stops the bot
answering anything at all. Analytics are worth a great deal less than
that, so a storage failure is logged and the frame carries on.
"""
sent_at_ms = invocation.sent_at_ms
day = day_of(sent_at_ms)
try:
store.record(
CommandEvent(
group_id=invocation.group_id,
user_uuid=invocation.user_uuid,
command=invocation.command,
args=args_token(invocation.args),
item_id=notes.item_id,
outcome=attempt.outcome,
error_class=attempt.error_class,
invoked_at_ms=sent_at_ms,
day=day,
local_hour=hour_of(sent_at_ms),
weekday=weekday_of(day),
handler_ms=attempt.handler_ms,
send_ms=notes.send_ms,
calls=tuple(notes.calls),
)
)
except Exception:
logger.exception("could not record the use of %s", invocation.command)

199
signal_bot/analytics/render.py
View File

@ -1,199 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""What !analityka says in the group.
Twelve lines is a cap, not a quota: a line with nothing to say is dropped
rather than printed empty, the same way the nightly report drops its
highlights. No ``ljust`` anywhere -- Signal renders a proportional font, so
padding lines up in a terminal and on nobody's phone.
Two rules about people, both inherited from ``DOCS-statistics.md``.
Only somebody whose display name has actually been seen may be named; the
group listing knows everyone else as a raw uuid or a phone number, and
printing either into a chat is a leak. And a per-person breakdown of
everybody's habits stays on the management page: the group gets a top three
and the asker's own favourite, which is what was decided when this was
specified.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from signal_bot.analytics import trend as trends
from signal_bot.stats import text
from signal_bot.stats.periods import pl_day_month
if TYPE_CHECKING:
from collections.abc import Mapping, Sequence
from signal_bot.analytics.demand import Demand
from signal_bot.analytics.hours import Peak
from signal_bot.analytics.scope import Window
from signal_bot.analytics.usage import CommandUse, PersonUse
MAX_LINES: Final = 12
TOP_COMMANDS: Final = 3
TOP_PEOPLE: Final = 3
HEADING: Final = "Analityka komend"
NOTHING_YET: Final = "Nikt jeszcze nie użył żadnej komendy."
DIRECTIONS: Final = {
trends.RISING: "użycie rośnie",
trends.FALLING: "użycie spada",
trends.FLAT: "użycie się trzyma",
}
def days_phrase(count: int) -> str:
"""A day count with its noun agreeing: 3 dni."""
return f"{count} {text.plural(count, 'dzień', 'dni', 'dni')}"
def times_phrase(count: int) -> str:
"""A count with its noun agreeing: 5 razy."""
return f"{count} {text.plural(count, 'raz', 'razy', 'razy')}"
def window_line(measured: Window) -> str:
"""What the numbers actually cover. Always printed, always first.
The bot is days old, not months. A reader who cannot see that is reading
a month panel drawn from three days as though it were a month.
"""
span = days_phrase(measured.days)
if measured.first_day == measured.last_day:
return f"Okno: {span} ({pl_day_month(measured.first_day)}), {times_phrase(measured.events)}"
return (
f"Okno: {span} ({pl_day_month(measured.first_day)}"
f" — {pl_day_month(measured.last_day)}), {times_phrase(measured.events)}"
)
def top_line(uses: Sequence[CommandUse]) -> str:
"""The three most used commands, with their counts."""
best = ", ".join(f"!{use.command} {use.uses}" for use in uses[:TOP_COMMANDS])
return f"Najczęściej: {best}"
def rarest_line(uses: Sequence[CommandUse]) -> str:
"""The least used command that anybody has used at all."""
quietest = uses[-1]
return f"Najrzadziej: !{quietest.command} — {times_phrase(quietest.uses)}"
def hours_line(peak: Peak) -> str:
"""When the group actually reaches for the bot."""
return f"Najgłośniejsza godzina: {peak.busiest:02d}:00 ({times_phrase(peak.busiest_uses)})"
def trend_line(direction: str) -> str:
"""The verdict, or nothing at all when it is too early to have one.
Returning "" rather than "za mało danych" is deliberate: the window line
above already says how many days there are, and repeating it as an
apology wastes one of twelve lines.
"""
return DIRECTIONS.get(direction, "")
def health_line(uses: Sequence[CommandUse]) -> str:
"""The command most in need of attention, if any is.
Refusals count as much as errors here. A command refused half the time
is not popular, it is mis-tuned, and from the group's side both read as
"it does not work".
"""
troubled = [use for use in uses if use.errors or use.refused]
if not troubled:
return ""
worst = max(troubled, key=lambda use: (use.errors + use.refused, use.command))
# "błędów: 3" rather than "3 błędy": a label and a number needs no
# agreement at all, and Polish agreement after a numeral has three forms
# whose rule reads the last two digits. Sidestepping it here is free.
parts = []
if worst.errors:
parts.append(f"błędów: {worst.errors}")
if worst.refused:
parts.append(f"odmów: {worst.refused}")
return f"Do poprawki: !{worst.command} ({', '.join(parts)})"
def demand_line(demand: Demand) -> str:
"""The trigger people keep almost typing, which wants another alias."""
if not demand.near_misses:
return ""
miss = demand.near_misses[0]
return f"Blisko trafień: {miss.target} — {times_phrase(miss.hits)}"
def people_line(people: Sequence[PersonUse], names: Mapping[str, str]) -> str:
"""The top three, and only those whose name the bot has actually seen."""
named = [(names[person.uuid], person.uses) for person in people if person.uuid in names]
if not named:
return ""
best = ", ".join(f"{name} {uses}" for name, uses in named[:TOP_PEOPLE])
return f"Kto najwięcej: {best}"
def mine_line(people: Sequence[PersonUse], uuid: str) -> str:
"""The asker's own favourite. Self-scope, so it names nobody else."""
for person in people:
if person.uuid == uuid and person.favourite:
return f"Twoja ulubiona: !{person.favourite} ({times_phrase(person.uses)})"
return ""
def share_line(commands: int, messages: int) -> str:
"""How much of the group's talking is talking to the bot."""
if not messages:
return ""
percent = round(100 * commands / messages)
return f"Komendy to {percent}% wiadomości w tej grupie"
@dataclass(frozen=True, slots=True)
class Summary:
"""Everything the reply needs, gathered before any of it is worded.
One object rather than ten arguments, because these always travel
together: the command builds it from the query layer and the renderer
turns it into Polish, and neither half needs to know the other.
"""
measured: Window
uses: Sequence[CommandUse]
people: Sequence[PersonUse]
names: Mapping[str, str]
peak: Peak
direction: str
demand: Demand
asker: str
commands: int
messages: int
def render(summary: Summary) -> str:
"""Build the reply, capped at twelve lines and never padded."""
if not summary.uses:
return NOTHING_YET
lines = [
HEADING,
window_line(summary.measured),
top_line(summary.uses),
rarest_line(summary.uses),
hours_line(summary.peak),
trend_line(summary.direction),
share_line(summary.commands, summary.messages),
health_line(summary.uses),
demand_line(summary.demand),
people_line(summary.people, summary.names),
mine_line(summary.people, summary.asker),
]
# Dropped, then capped. A line with nothing to say is not printed empty,
# so the cap bites on content rather than on blanks -- and the cap is
# what keeps a reply readable on a phone.
kept = [line for line in lines if line][:MAX_LINES]
return "\n".join(kept)

119
signal_bot/analytics/scope.py
View File

@ -1,119 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""What a question is being asked *about*, and which rows may answer it.
Every query in this package is a complete literal string. None of them is
built by concatenation or by an f-string, which is the only arrangement ruff
S608 accepts and this repository takes no suppressions -- ``forget_group``
spells its DELETEs out for the same reason.
"One group, or all of them" is therefore not a clause that gets appended. It
is written into every query once, as::
WHERE (:group IS NULL OR group_id = :group)
Binding ``:group`` to None makes the first half true and the filter vanish.
So the roll-up and the per-group answer run *the same SQL*, which is what
stops them drifting apart -- two queries written to agree eventually will
not.
The other shared decision here is the source filter. Three classes of row
live in ``command_events``:
* ``live`` -- observed as it happened; the only rows with real durations.
* ``journald`` -- reconstructed from the log, with a real hour taken from
the message it was matched to, but no duration and an outcome that was
inferred rather than seen.
* ``daily_uses`` -- a count with no hour at all, carrying ``UNKNOWN_HOUR``.
Counting all three is right for "how often was this used" and wrong for "at
what hour" or "how slow is it". Those two predicates are named here, once,
so that the exclusion is a shared constant rather than something six modules
each have to remember. ``stats/backfill.py`` records what forgetting it
costs: a synthetic hour reached ``favourite_hour`` and the answer was
quietly wrong.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
import sqlite3
# How long a person may go quiet before their next command starts a new
# session. Used only by burstiness and co-occurrence, the two questions that
# need "in one sitting" to mean anything.
#
# Ten minutes: comfortably longer than the pause between two commands
# somebody is idly firing off, far shorter than the gap between two separate
# times of day they picked their phone up.
SESSION_GAP_MS: Final = 10 * 60 * 1000
# Below this many days, a direction is not stated. The line is still drawn --
# the reader can see it -- but no arrow and no "rośnie"/"spada" verdict,
# because three points of a new bot's novelty curve are not a trend.
TREND_MIN_DAYS: Final = 14
@dataclass(frozen=True, slots=True)
class Scope:
"""One group, or all of them."""
group_id: str | None = None
def params(self, **extra: object) -> dict[str, object]:
"""The bound parameters every query in this package expects."""
return {"group": self.group_id, **extra}
@dataclass(frozen=True, slots=True)
class Window:
"""Which days an answer actually covers, and where its rows came from.
Carried on every response because this bot is days old, not months. A
month panel drawn from three days is not wrong, but a reader who cannot
see that it is three days will read it as though it were a month.
"""
first_day: str = ""
last_day: str = ""
days: int = 0
events: int = 0
live: int = 0
journald: int = 0
daily: int = 0
@property
def enough_for_a_trend(self) -> bool:
"""Whether a direction may be stated rather than merely plotted."""
return self.days >= TREND_MIN_DAYS
WINDOW_SQL: Final = """
SELECT MIN(day) AS first_day,
MAX(day) AS last_day,
COUNT(DISTINCT day) AS days,
COUNT(*) AS events,
SUM(source = 'live') AS live,
SUM(source = 'journald') AS journald,
SUM(source = 'daily_uses') AS daily
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
"""
def window(conn: sqlite3.Connection, scope: Scope) -> Window:
"""Measure what the data actually covers, before anything is drawn."""
row = conn.execute(WINDOW_SQL, scope.params()).fetchone()
if row is None or not row["events"]:
return Window()
return Window(
first_day=str(row["first_day"]),
last_day=str(row["last_day"]),
days=int(row["days"]),
events=int(row["events"]),
live=int(row["live"] or 0),
journald=int(row["journald"] or 0),
daily=int(row["daily"] or 0),
)

113
signal_bot/analytics/sessions.py
View File

@ -1,113 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family E, the behavioural half: sittings, bursts and what goes together.
These are the only questions that need a *trustworthy ordering in time*, so
they are the only ones restricted to ``live`` rows plus the journald ones
that carry a real hour. A ``daily_uses`` row is a count with a synthetic
timestamp built from midnight; treating those as a sequence would invent a
sitting in which somebody ran forty commands in forty milliseconds.
A session is one person's run of commands with no gap longer than
``SESSION_GAP_MS``. It exists to make two things sayable:
* **burstiness** -- how many commands a sitting contains. Three or four
means somebody is re-rolling for a better photo, which is a fact about
the pool rather than about them.
* **co-occurrence** -- which commands turn up in the same sitting, which is
what says whether two commands are alternatives or a habit.
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass
from itertools import combinations
from typing import TYPE_CHECKING, Final
from signal_bot.analytics.scope import SESSION_GAP_MS
if TYPE_CHECKING:
import sqlite3
from signal_bot.analytics.scope import Scope
# Only rows whose timestamp is a real moment. daily_uses rows are excluded
# by local_hour, which is the one column that distinguishes them.
ORDERED_SQL: Final = """
SELECT user_uuid, command, invoked_at_ms
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND local_hour >= 0
AND outcome != 'unknown_word'
ORDER BY user_uuid, invoked_at_ms
"""
@dataclass(frozen=True, slots=True)
class Sessions:
"""How the group actually uses the bot, sitting by sitting."""
count: int = 0
commands: int = 0
longest: int = 0
pairs: tuple[tuple[str, str, int], ...] = ()
repeats: tuple[tuple[str, int], ...] = ()
@property
def average_length(self) -> float:
"""Commands per sitting. One means nobody re-rolls."""
return self.commands / self.count if self.count else 0.0
def _split(rows: list[tuple[str, str, int]]) -> list[list[str]]:
"""Cut one person's ordered commands into sittings on the idle gap."""
sittings: list[list[str]] = []
current: list[str] = []
last_uuid = ""
last_at = 0
for uuid, command, at_ms in rows:
broken = uuid != last_uuid or at_ms - last_at > SESSION_GAP_MS
if broken and current:
sittings.append(current)
current = []
current.append(command)
last_uuid = uuid
last_at = at_ms
if current:
sittings.append(current)
return sittings
def collect(conn: sqlite3.Connection, scope: Scope) -> Sessions:
"""Split the history into sittings and describe what they contain."""
rows = [
(str(row["user_uuid"]), str(row["command"]), int(row["invoked_at_ms"]))
for row in conn.execute(ORDERED_SQL, scope.params())
]
sittings = _split(rows)
if not sittings:
return Sessions()
pairs: Counter[tuple[str, str]] = Counter()
repeats: Counter[str] = Counter()
for sitting in sittings:
# Sorted and de-duplicated, so (cat, dog) and (dog, cat) are one
# fact rather than two, and running !kot four times in a sitting
# does not make it co-occur with itself.
distinct = sorted(set(sitting))
for left, right in combinations(distinct, 2):
pairs[(left, right)] += 1
for command, times in Counter(sitting).items():
if times > 1:
repeats[command] += times - 1
return Sessions(
count=len(sittings),
commands=sum(len(sitting) for sitting in sittings),
longest=max(len(sitting) for sitting in sittings),
pairs=tuple(
(left, right, count)
for (left, right), count in sorted(pairs.items(), key=lambda p: (-p[1], p[0]))
),
repeats=tuple(sorted(repeats.items(), key=lambda pair: (-pair[1], pair[0]))),
)

187
signal_bot/analytics/trend.py
View File

@ -1,187 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family B, the calendar half: day series, months, and what is dying.
Day-granular, so every source counts -- a ``daily_uses`` row has no hour but
it does know its day, which is exactly what these questions need.
The direction of a trend is deliberately not always stated. ``Window``
carries the day count and ``TREND_MIN_DAYS`` is the threshold; below it the
series is returned and drawn but ``direction`` stays ``UNKNOWN``. A bot that
is three days old has a novelty curve, not a trend, and an arrow drawn from
three points is a claim the data cannot support.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from signal_bot.stats.periods import shift
if TYPE_CHECKING:
import sqlite3
from collections.abc import Sequence
from signal_bot.analytics.scope import Scope, Window
RISING: Final = "rising"
FALLING: Final = "falling"
FLAT: Final = "flat"
UNKNOWN: Final = "unknown"
# How much the second half must differ from the first to be called a
# direction rather than noise. A tenth: below that, on counts this small,
# one busy evening decides the answer.
TREND_MARGIN: Final = 0.10
# A command not used for this many days is called dying rather than merely
# quiet. Deliberately short, because the whole history is days long; it is
# a relative statement and the panel says so.
DYING_AFTER_DAYS: Final = 3
# A direction needs a before and an after to compare.
MIN_POINTS: Final = 2
BY_DAY_SQL: Final = """
SELECT day, COUNT(*) AS uses, COUNT(DISTINCT user_uuid) AS people
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome != 'unknown_word'
GROUP BY day
ORDER BY day
"""
BY_COMMAND_DAY_SQL: Final = """
SELECT command, day, COUNT(*) AS uses
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome != 'unknown_word'
GROUP BY command, day
ORDER BY command, day
"""
BY_MONTH_SQL: Final = """
SELECT substr(day, 1, 7) AS month, COUNT(*) AS uses, COUNT(DISTINCT day) AS days
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome != 'unknown_word'
GROUP BY month
ORDER BY month
"""
LAST_USED_SQL: Final = """
SELECT command, MAX(day) AS last_day, COUNT(*) AS uses
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'ok'
GROUP BY command
ORDER BY last_day, command
"""
@dataclass(frozen=True, slots=True)
class Dying:
"""A command that used to be used and has gone quiet."""
command: str
last_day: str
idle_days: int
uses: int
def by_day(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, int, int]]:
"""(day, uses, people) for every day anything happened."""
return [
(str(row["day"]), int(row["uses"]), int(row["people"]))
for row in conn.execute(BY_DAY_SQL, scope.params())
]
def dense_days(series: Sequence[tuple[str, int, int]]) -> list[tuple[str, int]]:
"""Fill in the days nothing happened on.
A sparse series draws a line straight over a silent day, which reads as
"steady" when the truth is "nobody used it". Built by stepping the date
rather than by adding 86_400 seconds, because Warsaw days are not all
the same length.
"""
if not series:
return []
filled: list[tuple[str, int]] = []
day = series[0][0]
last = series[-1][0]
counts = {entry[0]: entry[1] for entry in series}
while day <= last:
filled.append((day, counts.get(day, 0)))
day = shift(day, 1)
return filled
def by_command_day(conn: sqlite3.Connection, scope: Scope) -> dict[str, dict[str, int]]:
"""Per-command day series, for the sparklines."""
series: dict[str, dict[str, int]] = {}
for row in conn.execute(BY_COMMAND_DAY_SQL, scope.params()):
series.setdefault(str(row["command"]), {})[str(row["day"])] = int(row["uses"])
return series
def by_month(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, int, int]]:
"""(month, uses, days seen). The days matter: a partial month is not low."""
return [
(str(row["month"]), int(row["uses"]), int(row["days"]))
for row in conn.execute(BY_MONTH_SQL, scope.params())
]
def direction(series: Sequence[tuple[str, int]], measured: Window) -> str:
"""Whether use is rising, falling, flat -- or too early to say.
Compares the mean of the second half against the first. Not a
least-squares fit: on a dozen points a fit is no more truthful and far
harder to explain in a chat message.
"""
if not measured.enough_for_a_trend or len(series) < MIN_POINTS:
return UNKNOWN
# MIN_POINTS above guarantees half >= 1, so neither slice is empty and
# neither division below is by zero.
half = len(series) // 2
first = [count for _day, count in series[:half]]
second = [count for _day, count in series[half:]]
before = sum(first) / len(first)
after = sum(second) / len(second)
if before == 0:
return RISING if after > 0 else FLAT
change = (after - before) / before
if change > TREND_MARGIN:
return RISING
if change < -TREND_MARGIN:
return FALLING
return FLAT
def dying(conn: sqlite3.Connection, scope: Scope, today: str) -> list[Dying]:
"""Commands that worked, were used, and then stopped being used."""
found: list[Dying] = []
for row in conn.execute(LAST_USED_SQL, scope.params()):
last_day = str(row["last_day"])
idle = _days_between(last_day, today)
if idle >= DYING_AFTER_DAYS:
found.append(
Dying(
command=str(row["command"]),
last_day=last_day,
idle_days=idle,
uses=int(row["uses"]),
)
)
return sorted(found, key=lambda entry: (-entry.idle_days, entry.command))
def _days_between(start: str, end: str) -> int:
"""Whole calendar days from one day to another, never negative."""
days = 0
day = start
while day < end:
day = shift(day, 1)
days += 1
return days

197
signal_bot/analytics/usage.py
View File

@ -1,197 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Family A: who uses which commands, how often, and which nobody uses.
Counts include every source. "How many times was !papież used" is answered
by all three classes of row -- a reconstructed invocation still happened.
Only the questions that need an hour or a duration narrow to ``live``, and
those live in ``hours`` and ``health``.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
import sqlite3
from collections.abc import Sequence
from signal_bot.analytics.scope import Scope
@dataclass(frozen=True, slots=True)
class CommandUse:
"""One command, and everything countable about it."""
command: str
uses: int
people: int
ok: int
refused: int
errors: int
first_day: str
last_day: str
@property
def failure_rate(self) -> float:
"""Share of attempts that crashed. Refusals are not failures."""
return self.errors / self.uses if self.uses else 0.0
@property
def refusal_rate(self) -> float:
"""Share of attempts a gate turned away.
High is not popular, it is mis-tuned: it means people keep asking
and keep being told no.
"""
return self.refused / self.uses if self.uses else 0.0
@dataclass(frozen=True, slots=True)
class PersonUse:
"""One person's total, by uuid. Names are attached by the caller."""
uuid: str
uses: int
commands: int
favourite: str
BY_COMMAND_SQL: Final = """
SELECT command,
COUNT(*) AS uses,
COUNT(DISTINCT user_uuid) AS people,
SUM(outcome = 'ok') AS ok,
SUM(outcome LIKE 'refused_%') AS refused,
SUM(outcome = 'error') AS errors,
MIN(day) AS first_day,
MAX(day) AS last_day
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome != 'unknown_word'
GROUP BY command
ORDER BY uses DESC, command
"""
BY_PERSON_SQL: Final = """
SELECT user_uuid,
COUNT(*) AS uses,
COUNT(DISTINCT command) AS commands
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'ok'
GROUP BY user_uuid
ORDER BY uses DESC, user_uuid
"""
FAVOURITE_SQL: Final = """
SELECT user_uuid, command, COUNT(*) AS uses
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'ok'
GROUP BY user_uuid, command
ORDER BY user_uuid, uses DESC, command
"""
MATRIX_SQL: Final = """
SELECT user_uuid, command, COUNT(*) AS uses
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome = 'ok'
GROUP BY user_uuid, command
ORDER BY user_uuid, command
"""
TOTALS_SQL: Final = """
SELECT COUNT(*) AS attempts,
SUM(outcome = 'ok') AS ok,
COUNT(DISTINCT user_uuid) AS people,
COUNT(DISTINCT command) AS commands
FROM command_events
WHERE (:group IS NULL OR group_id = :group)
AND outcome != 'unknown_word'
"""
def by_command(conn: sqlite3.Connection, scope: Scope) -> list[CommandUse]:
"""Every command that has ever been attempted, most used first."""
return [
CommandUse(
command=str(row["command"]),
uses=int(row["uses"]),
people=int(row["people"]),
ok=int(row["ok"] or 0),
refused=int(row["refused"] or 0),
errors=int(row["errors"] or 0),
first_day=str(row["first_day"]),
last_day=str(row["last_day"]),
)
for row in conn.execute(BY_COMMAND_SQL, scope.params())
]
def _favourites(conn: sqlite3.Connection, scope: Scope) -> dict[str, str]:
"""Each person's most-used command.
The query orders by uses within each person, so the first row seen for
a uuid is the answer and the rest are skipped -- cheaper than a window
function and portable to the ancient sqlite a container might carry.
"""
best: dict[str, str] = {}
for row in conn.execute(FAVOURITE_SQL, scope.params()):
best.setdefault(str(row["user_uuid"]), str(row["command"]))
return best
def by_person(conn: sqlite3.Connection, scope: Scope) -> list[PersonUse]:
"""Who uses the bot, by uuid and most active first.
Deliberately returns uuids. Turning one into a name is the caller's job
and a different rule in each place: the group chat may name only people
whose display name has been seen, the management page may show
everyone, and neither may print a raw uuid or a phone number.
"""
favourites = _favourites(conn, scope)
return [
PersonUse(
uuid=str(row["user_uuid"]),
uses=int(row["uses"]),
commands=int(row["commands"]),
favourite=favourites.get(str(row["user_uuid"]), ""),
)
for row in conn.execute(BY_PERSON_SQL, scope.params())
]
def matrix(conn: sqlite3.Connection, scope: Scope) -> list[tuple[str, str, int]]:
"""Every (person, command, count) triple, for the heatmap on the page.
Never posted into a group -- a per-person breakdown of everybody's
habits is a management-page thing.
"""
return [
(str(row["user_uuid"]), str(row["command"]), int(row["uses"]))
for row in conn.execute(MATRIX_SQL, scope.params())
]
def totals(conn: sqlite3.Connection, scope: Scope) -> dict[str, int]:
"""Attempts, successes, distinct people and distinct commands."""
row = conn.execute(TOTALS_SQL, scope.params()).fetchone()
return {
"attempts": int(row["attempts"] or 0),
"ok": int(row["ok"] or 0),
"people": int(row["people"] or 0),
"commands": int(row["commands"] or 0),
}
def never_used(known: Sequence[str], used: Sequence[CommandUse]) -> list[str]:
"""Registered commands with no recorded attempt at all.
Takes the registry's list rather than reading it, because which commands
exist depends on the group -- the album's are registered only where the
album is -- and this module must not know that.
"""
seen = {use.command for use in used}
return sorted(name for name in known if name not in seen)

150
signal_bot/analytics/uses.py
View File

@ -1,150 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Writing the command history: one row per attempt, and what it cost.
Shares the ``Store`` connection exactly as ``EventStore`` does, so a command
and the message that triggered it are committed by the same connection in
the same order they happened.
The column set is pinned by ``test_command_events`` for the reason
``message_events``' is: this table sits beside a public repository that
leaked real names once, and a column added quietly is how the next one would
arrive. Nothing here holds free text. ``command`` is a registry name,
``args`` is a resolved token, ``item_id`` is a public identifier and
``error_class`` is an exception's class name.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Final
if TYPE_CHECKING:
import sqlite3
from collections.abc import Sequence
from signal_bot.analytics.probe import UpstreamCall
OUTCOME_OK: Final = "ok"
OUTCOME_COOLDOWN: Final = "refused_cooldown"
OUTCOME_DAILY: Final = "refused_daily"
OUTCOME_VOTE: Final = "refused_vote"
OUTCOME_OFF: Final = "refused_off"
OUTCOME_ERROR: Final = "error"
OUTCOME_UNKNOWN: Final = "unknown_word"
SOURCE_LIVE: Final = "live"
SOURCE_JOURNAL: Final = "journald"
SOURCE_DAILY: Final = "daily_uses"
# What ``args`` holds when a command took arguments that resolve to nothing
# nameable. Never the words themselves: a user-typed argument column is the
# exact shape that put real names in this repository once.
ARGS_PRESENT: Final = "+args"
@dataclass(frozen=True, slots=True)
class CommandEvent:
"""One attempt to run one command, whether or not it ran."""
group_id: str
user_uuid: str
command: str
outcome: str
invoked_at_ms: int
day: str
local_hour: int
weekday: int
args: str = ""
item_id: str | None = None
error_class: str = ""
# None rather than 0: a backfilled row has no duration, and zero
# milliseconds is a lie that reads as data and would drag a p50 down.
# The lesson is already written into backfill.UNKNOWN_HOUR's comment.
handler_ms: int | None = None
send_ms: int | None = None
source: str = SOURCE_LIVE
calls: Sequence[UpstreamCall] = field(default_factory=tuple)
class CommandEventStore:
"""Reads and writes the command history. Shares the Store's connection."""
def __init__(self, conn: sqlite3.Connection) -> None:
"""Wrap an open connection whose schema has already been applied."""
self._conn = conn
@property
def connection(self) -> sqlite3.Connection:
"""The shared connection, for the query modules in this package."""
return self._conn
def record(self, event: CommandEvent) -> None:
"""Append one attempt and the upstream calls it made.
``ON CONFLICT DO NOTHING`` against the natural key is what makes a
signal-cli redelivery harmless, the same property ``message_events``
gets from its primary key -- ``invoked_at_ms`` is the envelope's own
timestamp, so a replayed frame collides with the row it already
wrote instead of adding a second one.
It is also what makes the backfill re-runnable, which is why the
backfill needs no delete-then-rebuild pass and cannot lose rows to
a journal that has since been vacuumed.
"""
self._conn.execute(
"""
INSERT INTO command_events (
group_id, user_uuid, command, args, item_id, outcome,
error_class, invoked_at_ms, day, local_hour, weekday,
handler_ms, send_ms, source
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(group_id, user_uuid, command, invoked_at_ms, source)
DO NOTHING
""",
(
event.group_id,
event.user_uuid,
event.command,
event.args,
event.item_id,
event.outcome,
event.error_class,
event.invoked_at_ms,
event.day,
event.local_hour,
event.weekday,
event.handler_ms,
event.send_ms,
event.source,
),
)
self._record_calls(event)
self._conn.commit()
def _record_calls(self, event: CommandEvent) -> None:
"""Append the upstream calls one invocation made.
Keyed on (group, timestamp, seq) rather than on the parent's rowid.
``lastrowid`` is unusable here: ``ON CONFLICT DO NOTHING`` leaves it
holding whatever the previous successful insert returned, so a
replayed frame would file its calls under an unrelated invocation.
"""
for seq, call in enumerate(event.calls):
self._conn.execute(
"""
INSERT INTO upstream_calls (
group_id, invoked_at_ms, command, host, status,
duration_ms, seq
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(group_id, invoked_at_ms, seq) DO NOTHING
""",
(
event.group_id,
event.invoked_at_ms,
event.command,
call.host,
call.status,
call.ms,
seq,
),
)

96
signal_bot/analytics_command.py
View File

@ -1,96 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""!analityka -- what the group has actually been asking the bot for.
Registered here rather than in ``stats/commands.py``, which is a few lines
under the 250 cap already, and because this reads a different table:
``stats`` is the *message* history, this is the *command* history.
The reply is text only, in Polish, capped at twelve lines. The charts live
on the management page; a rendered image in the chat would mean a new
dependency and, per ``DOCS-latency.md``, would turn a 250 ms text reply into
a three-second one for a picture nobody can zoom into.
Like ``!pomoc``, the registry is read when the command *runs* rather than
when it is registered -- at registration time the album's commands do not
exist yet, and those are exactly the triggers a near miss might be against.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Final
from signal_bot.analytics import adoption, demand, hours, render, trend, usage
from signal_bot.analytics.scope import Scope, window
from signal_bot.animals import load_aliases
if TYPE_CHECKING:
from collections.abc import Sequence
from signal_bot.context import CommandContext
from signal_bot.decorators import Decorator
from signal_bot.registry import Command, CommandRegistry, Handler
COMMAND_NAME: Final = "command_stats"
DESCRIPTION: Final = "kto ile używa bota i które komendy żyją"
def triggers_for(commands: Sequence[Command], group_id: str) -> tuple[str, ...]:
"""Every trigger this group could plausibly have meant.
Scoped to the group, and that scoping is the point: the album's triggers
are real people's names and exist only where the album does. Reporting a
near miss against one outside its group would announce that the album
exists, which is the same leak the dispatcher refuses at resolution.
"""
return tuple(
trigger
for command in commands
if command.allowed_in(group_id)
for trigger in command.triggers
)
def summarise(ctx: CommandContext, commands: Sequence[Command]) -> render.Summary:
"""Gather every number the reply needs, from the group that asked.
Scoped to one group like every other statistic here: a test group's
numbers must never turn up in the production group's answer.
"""
conn = ctx.store.commands.connection
scope = Scope(ctx.group_id)
measured = window(conn, scope)
series = trend.dense_days(trend.by_day(conn, scope))
share = adoption.share_of_chat(conn, scope)
return render.Summary(
measured=measured,
uses=usage.by_command(conn, scope),
people=usage.by_person(conn, scope),
# Only people the bot has actually seen a name for. Everyone else is
# counted and left unnamed: the roster knows them as a raw uuid or a
# phone number, and printing either into a chat is a leak.
names={
person.uuid: person.display_name for person in ctx.store.events.people(ctx.group_id)
},
peak=hours.peak(hours.by_hour(conn, scope)),
direction=trend.direction(series, measured),
demand=demand.collect(conn, scope, triggers_for(commands, ctx.group_id)),
asker=ctx.message.source_uuid,
commands=share.commands,
messages=share.messages,
)
def register_analytics_command(
registry: CommandRegistry,
*,
decorate: Decorator | None = None,
aliases: dict[str, list[str]] | None = None,
) -> None:
"""Register !analityka under every trigger aliases.json gives it."""
table = load_aliases() if aliases is None else aliases
async def handler(ctx: CommandContext) -> None:
await ctx.reply(render.render(summarise(ctx, registry.commands)))
wrapped: Handler = decorate(handler) if decorate else handler
registry.register(COMMAND_NAME, table[COMMAND_NAME], wrapped, DESCRIPTION)

Some files were not shown because too many files have changed in this diff Show More