mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-11 13:37:32 +01:00
Krzysztof kuhy Rudnicki
08599e6056
!sztuka posts a work of art, skewed 60% Polish; !wiersz posts one short public-domain Polish poem, whole. Both follow the rule the rest of the bot already keeps: the repository commits identifiers, and the words arrive over the wire when somebody asks. !wiersz was written the other way first, and the gate caught it. A pack of short public-domain poems was harvested, name-filtered and committed, and check_personal_data.sh refused it: three lines carried a real group member's given name, and two still did after a filter that had discarded 31% of the corpus. That is structural, not a tuning problem -- Polish verse capitalises every line, so a line-initial capital carries no signal, and that is exactly where a name sits. The pack now holds Wolne Lektury slugs; the poem, its title and its author are fetched at post time and appear nowhere in git. It is also the better version: with no name filter there are no false positives, so the pool is larger and the attribution is a full name again. The gate then refused the art category list too, over a painter whose given name is somebody's. That is the popes.json case exactly, so it takes the same answer: a file-scoped allowlist entry, still refused everywhere else. The Commons harvest is now three modules by what they are, not by where the line count fell: commons_api is the conversation with Commons, commons_harvest turns categories into a pool of licensed page ids, commons_cache makes an interrupted run resumable. Along the way it gained licences in the same request that lists a category (the second pass was 1,500 requests at 2.3s each), maxlag instead of a guessed sleep, 5xx retries because one 504 used to destroy a whole run, and Wikidata for works by creators of Polish citizenship: the Polish pool grew 15,221 -> 20,092. Batching categories through CirrusSearch looked like a further 6x and was measured out again: 2,404 files where the authoritative enumeration returns 2,429, a different ~1% missing on every run. The offline run is 19s where it was four minutes, almost all of it the deafness gate waiting out three thirty-second liveness probes to prove something provable in three; the tick is configuration now, and e2e enforces a 40s budget on itself so that never silently comes back.
95 lines
3.4 KiB
Python
Executable File
95 lines
3.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""Regenerate ``signal_bot/traps.json`` from Wikimedia Commons.
|
|
|
|
Run this by hand; the output is committed, so the bot never searches Commons
|
|
at the moment somebody types ``!pułapka``.
|
|
|
|
**The file holds page ids, not filenames, and that is load-bearing.** Commons
|
|
filenames are free text carrying people's names -- the harvest turns up things
|
|
like ``"Fishing-Boat with Fishermen" - Vasileios Chatzis.jpg``. Committing two
|
|
thousand of those would eventually collide with a label somebody minted from
|
|
the chat with ``!add_photo``, and ``scripts/check_personal_data.py`` would then
|
|
refuse every commit for good, over data nobody can edit out by hand. An
|
|
integer cannot carry a name. ``prop=imageinfo`` accepts ``pageids=`` and puts
|
|
``thumburl`` at the same place ``titles=`` does, so nothing downstream cares.
|
|
|
|
The API plumbing -- the retry loop, the continuation handling, the licence
|
|
filter and the depth-of-one rule -- lives in ``commons_api.py``, which
|
|
``gen_art.py`` shares.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
import commons_api
|
|
import commons_harvest
|
|
|
|
CATEGORIES_FILE: Final = Path(__file__).with_name("trap_categories.json")
|
|
|
|
USER_AGENT: Final = "signal-bot traps generator (https://github.com/kuhyx/signal-bot)"
|
|
|
|
# A harvest smaller than this is a truncated one, and committing it would
|
|
# quietly shrink the command's whole world.
|
|
MINIMUM: Final = 500
|
|
|
|
|
|
def categories() -> tuple[tuple[str, ...], frozenset[str], tuple[str, ...]]:
|
|
"""Return the seeds, the pruned set and the pruned patterns."""
|
|
document = json.loads(CATEGORIES_FILE.read_text(encoding="utf-8"))
|
|
return (
|
|
tuple(document["seeds"]),
|
|
frozenset(document["excluded"]),
|
|
tuple(document["prune_patterns"]),
|
|
)
|
|
|
|
|
|
def harvest() -> dict[int, str]:
|
|
"""Page id to title, for every usable file one step from the seeds."""
|
|
seeds, excluded, patterns = categories()
|
|
commons_api.verify_seeds(seeds, USER_AGENT)
|
|
return commons_harvest.walk(seeds, excluded, patterns, USER_AGENT)
|
|
|
|
|
|
def _rejection(traps: dict[int, str]) -> str:
|
|
"""Return why this harvest must not be written, or an empty string."""
|
|
if len(traps) < MINIMUM:
|
|
return f"got {len(traps)} traps, expected at least {MINIMUM}"
|
|
return ""
|
|
|
|
|
|
def main() -> int:
|
|
"""Write traps.json, refusing to write a harvest that failed its own checks."""
|
|
parser = argparse.ArgumentParser(description="Regenerate signal_bot/traps.json.")
|
|
parser.add_argument(
|
|
"--titles",
|
|
action="store_true",
|
|
help="print every kept title; the committed file is integers, so this is the review",
|
|
)
|
|
arguments = parser.parse_args()
|
|
|
|
traps = commons_harvest.licensed(harvest(), USER_AGENT)
|
|
rejection = _rejection(traps)
|
|
if rejection:
|
|
sys.stderr.write(f"refusing to write: {rejection}\n")
|
|
return 1
|
|
|
|
if arguments.titles:
|
|
for page in sorted(traps):
|
|
sys.stdout.write(f"{page}\t{traps[page]}\n")
|
|
|
|
out = Path(__file__).resolve().parent.parent / "signal_bot" / "traps.json"
|
|
document = {"source": str(CATEGORIES_FILE.name), "traps": sorted(traps)}
|
|
out.write_text(json.dumps(document, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
|
sys.stdout.write(f"wrote {len(traps)} traps to {out}\n")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|