mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-10 19:27:20 +01:00
!sztuka posts a work of art, skewed 60% Polish; !wiersz posts one short public-domain Polish poem, whole. Both follow the rule the rest of the bot already keeps: the repository commits identifiers, and the words arrive over the wire when somebody asks. !wiersz was written the other way first, and the gate caught it. A pack of short public-domain poems was harvested, name-filtered and committed, and check_personal_data.sh refused it: three lines carried a real group member's given name, and two still did after a filter that had discarded 31% of the corpus. That is structural, not a tuning problem -- Polish verse capitalises every line, so a line-initial capital carries no signal, and that is exactly where a name sits. The pack now holds Wolne Lektury slugs; the poem, its title and its author are fetched at post time and appear nowhere in git. It is also the better version: with no name filter there are no false positives, so the pool is larger and the attribution is a full name again. The gate then refused the art category list too, over a painter whose given name is somebody's. That is the popes.json case exactly, so it takes the same answer: a file-scoped allowlist entry, still refused everywhere else. The Commons harvest is now three modules by what they are, not by where the line count fell: commons_api is the conversation with Commons, commons_harvest turns categories into a pool of licensed page ids, commons_cache makes an interrupted run resumable. Along the way it gained licences in the same request that lists a category (the second pass was 1,500 requests at 2.3s each), maxlag instead of a guessed sleep, 5xx retries because one 504 used to destroy a whole run, and Wikidata for works by creators of Polish citizenship: the Polish pool grew 15,221 -> 20,092. Batching categories through CirrusSearch looked like a further 6x and was measured out again: 2,404 files where the authoritative enumeration returns 2,429, a different ~1% missing on every run. The offline run is 19s where it was four minutes, almost all of it the deafness gate waiting out three thirty-second liveness probes to prove something provable in three; the tick is configuration now, and e2e enforces a 40s budget on itself so that never silently comes back.
145 lines
5.4 KiB
Python
145 lines
5.4 KiB
Python
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""Outbound HTTP for the image commands.
|
|
|
|
Every outbound request goes through this one injectable client, so the tests
|
|
and the offline environment can serve fixtures without the bot knowing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
from typing import TYPE_CHECKING, Final
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Sequence
|
|
|
|
import httpx
|
|
|
|
|
|
class ImageFetchError(RuntimeError):
|
|
"""Raised when an upstream image API did not yield a usable image."""
|
|
|
|
|
|
class ImageFetcher:
|
|
"""Resolves an image URL from a JSON API and returns it base64-encoded."""
|
|
|
|
def __init__(self, client: httpx.AsyncClient) -> None:
|
|
"""Store the injected HTTP client."""
|
|
self._client = client
|
|
|
|
async def fetch_base64(
|
|
self,
|
|
api_url: str,
|
|
url_path: Sequence[str | int],
|
|
rewrite: tuple[str, str] | None = None,
|
|
) -> str:
|
|
"""Fetch an image and return it base64-encoded, ready for Signal."""
|
|
return base64.b64encode(await self.fetch_bytes(api_url, url_path, rewrite)).decode("ascii")
|
|
|
|
async def fetch_bytes(
|
|
self,
|
|
api_url: str,
|
|
url_path: Sequence[str | int],
|
|
rewrite: tuple[str, str] | None = None,
|
|
params: dict[str, str] | None = None,
|
|
) -> bytes:
|
|
"""Fetch an image and return its raw bytes.
|
|
|
|
Args:
|
|
api_url: JSON endpoint that names an image.
|
|
url_path: Keys/indices to walk in the JSON to reach the image URL.
|
|
rewrite: Optional (find, replace) pair applied to the URL found
|
|
there. Some APIs name a thumbnail and leave the full-size
|
|
photo one substitution away -- iNaturalist's observation JSON
|
|
gives a 75px `/square.` URL whose `/large.` sibling is the
|
|
real photo -- and a substitution keeps that upstream quirk as
|
|
data next to the endpoint rather than as a branch in here.
|
|
|
|
params: Query string for the JSON endpoint, when it is easier to
|
|
pass than to embed -- the Commons API takes a title that has
|
|
to be percent-encoded, the animal APIs are fixed URLs.
|
|
|
|
Returns:
|
|
The image bytes.
|
|
|
|
Raises:
|
|
ImageFetchError: The JSON did not contain a usable URL.
|
|
"""
|
|
image_url = _walk(await self.fetch_json(api_url, params), url_path)
|
|
if rewrite is not None:
|
|
image_url = image_url.replace(*rewrite)
|
|
return await self.fetch_image(image_url)
|
|
|
|
async def fetch_json(self, api_url: str, params: dict[str, str] | None = None) -> object:
|
|
"""Fetch one JSON document from an API.
|
|
|
|
Separate from fetch_bytes because a Commons answer carries more than
|
|
the picture: !pułapka needs the description in the same reply, and
|
|
asking twice would be both wasteful and a chance for the two halves
|
|
to disagree.
|
|
"""
|
|
response = await self._client.get(api_url, params=params)
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
async def fetch_text(self, url: str) -> str:
|
|
"""Fetch one plain-text document.
|
|
|
|
!wiersz is the only command whose upstream serves neither JSON nor an
|
|
image: Wolne Lektury hands back the poem as text/plain. It goes
|
|
through this client like everything else, so the offline environment
|
|
and the tests serve it the same way they serve a cat.
|
|
"""
|
|
response = await self._client.get(url)
|
|
response.raise_for_status()
|
|
return response.text
|
|
|
|
async def fetch_image(self, image_url: str) -> bytes:
|
|
"""Fetch the bytes at an already-resolved image URL.
|
|
|
|
These bytes are never written to a path built from the remote URL:
|
|
the original bot named a local file after the URL's last path
|
|
segment and wrote it into the process CWD, which is both path
|
|
traversal and unbounded. portraits.py does cache to disk, but under
|
|
a name it derives from the pope's ordinal, never from upstream.
|
|
"""
|
|
image = await self._client.get(image_url)
|
|
image.raise_for_status()
|
|
return image.content
|
|
|
|
|
|
_MISSING: Final = object()
|
|
|
|
|
|
def _in_range(items: list[object], index: int) -> bool:
|
|
"""True when index addresses a real element, counting from either end."""
|
|
return -len(items) <= index < len(items)
|
|
|
|
|
|
def _step(node: object, key: str | int, path: Sequence[str | int]) -> object:
|
|
"""Descend one level into a JSON document.
|
|
|
|
Narrows explicitly instead of indexing blindly, so a malformed upstream
|
|
response raises ImageFetchError rather than a TypeError from deeper in.
|
|
"""
|
|
found: object = _MISSING
|
|
if isinstance(key, str) and isinstance(node, dict):
|
|
found = node.get(key, _MISSING)
|
|
elif isinstance(key, int) and isinstance(node, list) and _in_range(node, key):
|
|
found = node[key]
|
|
if found is _MISSING:
|
|
msg = f"image API response has no element at {path!r}"
|
|
raise ImageFetchError(msg)
|
|
return found
|
|
|
|
|
|
def _walk(document: object, path: Sequence[str | int]) -> str:
|
|
"""Walk a JSON document along path and return the string found there."""
|
|
node: object = document
|
|
for key in path:
|
|
node = _step(node, key, path)
|
|
if not isinstance(node, str) or not node:
|
|
msg = f"image API element at {path!r} is not a URL: {node!r}"
|
|
raise ImageFetchError(msg)
|
|
return node
|