signal-bot/tool/commons_api.py

#!/usr/bin/env python3
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Talking to Wikimedia Commons, for the generators that harvest from it.

Extracted from ``gen_traps.py`` when ``gen_art.py`` arrived and would
otherwise have carried a second copy of the same retry loop, the same
continuation handling and the same licence filter. Every constant here was
measured by that first generator; the comments say what by.

Nothing in this module knows what a trap or a painting is. It knows how to
ask Commons a question without being rate-limited into silence, and how to
tell whether a file may be reposted.
"""

from __future__ import annotations

import time
from typing import Any, Final

import httpx

API: Final = "https://commons.wikimedia.org/w/api.php"

# The extension is what tells Signal how to render the attachment, so an
# unknown one is dropped rather than guessed. Same set as portraits.SUFFIXES.
SUFFIXES: Final = frozenset({".jpg", ".jpeg", ".png", ".gif"})

# Commons hosts non-free content too. !gekon filters licences at its query and
# endpoints.py calls that filter load-bearing; this is the same idea, on the
# only field Commons exposes for it. Matched case-folded, by prefix.
LICENCES: Final = ("public domain", "cc0", "cc by", "cc-by", "no restrictions")

# imageinfo takes fifty page ids per request, joined with "|". A comma is
# rejected outright ("badinteger"), which is worth writing down because the
# error names the parameter rather than the separator.
BATCH: Final = 50

# Wikimedia's etiquette asks for requests in series, and a strictly serial
# walk of a hundred-thousand-file harvest is the best part of an hour. Three
# is the compromise: enough to hide the round trip, few enough to stay a
# polite read against a project that serves everybody. The 429 handling in
# get() is the safety net rather than the plan -- if Commons does push back,
# every worker backs off exponentially and the harvest slows instead of
# failing. Only the per-category *file* queries run this way; claiming a
# category still happens serially, so `seen` and the pool ordering it
# guarantees are untouched.
WORKERS: Final = 3


TOO_MANY_REQUESTS: Final = 429
SERVER_ERROR: Final = 500
RETRIES: Final = 6
PAUSE: Final = 0.4

# Wikimedia's documented way to be a good citizen, and better than guessing a
# sleep: the server reports its own replication lag and refuses work when it
# is behind, so the throttle is feedback instead of superstition. Five seconds
# is the value their API docs give for interactive tools.
MAXLAG: Final = "5"


def get(params: dict[str, str], user_agent: str) -> dict[str, Any]:
    """One API call, retried with a widening pause when Commons says slow down.

    Commons answers 429 after a few hundred requests even at a steady 0.4s,
    and a harvest that silently loses a category is worse than a slow one.

    Args:
        params: The query, minus the action/format boilerplate.
        user_agent: Wikimedia's policy requires a descriptive agent with a
            contact URL; the generic client default is rate-limited hard and
            may be blocked outright.

    Returns:
        The decoded response body.

    Raises:
        RuntimeError: If Commons answered 429 to every attempt.
    """
    query = {
        "action": "query",
        "format": "json",
        "formatversion": "2",
        "maxlag": MAXLAG,
        **params,
    }
    for attempt in range(RETRIES):
        time.sleep(PAUSE)
        try:
            response = httpx.get(
                API, params=query, headers={"User-Agent": user_agent}, timeout=120.0
            )
        except httpx.TransportError:
            # A dropped connection is not an answer; treat it like a 5xx.
            time.sleep(2.0**attempt)
            continue
        # 429 is Commons asking for less; 5xx is Commons failing under the
        # weight of a generator+imageinfo query, which is expensive enough to
        # time out on the biggest categories. Both are transient and both
        # used to be fatal -- a single 504 on Category:Drawings threw away a
        # nine-minute harvest, which is what put this branch here.
        if response.status_code == TOO_MANY_REQUESTS or response.status_code >= SERVER_ERROR:
            time.sleep(2.0**attempt)
            continue
        response.raise_for_status()
        document: dict[str, Any] = response.json()
        # maxlag is reported in-band with HTTP 200, not as a status code.
        if isinstance(document.get("error"), dict) and document["error"].get("code") == "maxlag":
            time.sleep(2.0**attempt)
            continue
        return document
    msg = f"Commons kept refusing {params}"
    raise RuntimeError(msg)


def members(category: str, kind: str, user_agent: str) -> list[dict[str, Any]]:
    """Every member of one category, following continuations."""
    found: list[dict[str, Any]] = []
    carry_on: str | None = None
    while True:
        params = {
            "list": "categorymembers",
            "cmtitle": category,
            "cmtype": kind,
            "cmlimit": "500",
        }
        if carry_on is not None:
            params["cmcontinue"] = carry_on
        document = get(params, user_agent)
        found += document.get("query", {}).get("categorymembers", [])
        carry_on = document.get("continue", {}).get("cmcontinue")
        if carry_on is None:
            return found


def licence_of(page: dict[str, Any]) -> str:
    """The licence short name for one imageinfo page, or "" if it has none.

    Every field is checked rather than indexed, because MediaWiki serialises
    an **empty map as an empty list** -- a PHP array quirk that
    formatversion=2 does not fully hide. A file whose extmetadata is empty
    arrives as ``"extmetadata": []``, and calling .get() on that threw
    AttributeError forty thousand files into a harvest and lost the run. The
    traps corpus happened not to contain one; the art corpus does.
    """
    info = page.get("imageinfo")
    if not isinstance(info, list) or not info or not isinstance(info[0], dict):
        return ""
    meta = info[0].get("extmetadata")
    if not isinstance(meta, dict):
        return ""
    short = meta.get("LicenseShortName")
    if not isinstance(short, dict):
        return ""
    return str(short.get("value", ""))


def verify_seeds(seeds: tuple[str, ...], user_agent: str) -> None:
    """Fail before harvesting if any seed category does not exist.

    A category that is not there is not an error to Commons -- it is an empty
    answer -- so a plausible-looking typo removes a whole subject silently.
    Five of the trap generator's first-draft names were wrong this way, and
    fourteen of the art generator's were.

    Raises:
        RuntimeError: If any seed is missing, naming all of them at once.
    """
    missing: list[str] = []
    for start in range(0, len(seeds), BATCH):
        document = get({"titles": "|".join(seeds[start : start + BATCH])}, user_agent)
        missing += [
            str(page["title"])
            for page in document.get("query", {}).get("pages", [])
            if page.get("missing")
        ]
    if missing:
        msg = f"these seed categories do not exist on Commons: {sorted(missing)}"
        raise RuntimeError(msg)