signal-bot/scripts/personal_data_sources.py

#!/usr/bin/env python3
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""Where the forbidden words come from -- all of it outside this repository.

Split out of check_personal_data.py at the 250-line cap, and the seam is the
honest one: this half knows the private data, that half knows nothing but how
to search. A list of real names kept *in* the repository to check against
would be the leak the check exists to prevent, so nothing here is ever
written down -- it is read at run time from the photo pack, the message
database and the local .env, and on a machine with none of those it simply
comes back empty.
"""

from __future__ import annotations

import base64
import json
import os
import sqlite3
from pathlib import Path
from typing import Any, Final

REPO: Final = Path(__file__).resolve().parent.parent

# Three, because real first names are three letters long and one of this
# group's is. Two would match inside half the words in the repository; three
# needs a handful of allowlist entries for the given names that are also
# popes, which is a cost worth paying to catch a real one.
MIN_NAME: Final = 3

# How much of a group id has to appear before it counts as that group. Eight
# base64 characters are 48 bits: unique enough that a coincidence is not
# worth worrying about, short enough to catch a truncated quotation.
PREFIX: Final = 8


def _pack_names(directory: Path) -> set[str]:
    """Every trigger word in a photo pack manifest -- i.e. every name."""
    document = _pack_document(directory / "people.json")
    entries = [*document.get("people", []), *document.get("combos", [])]
    return {
        str(trigger).lstrip("!").strip()
        for entry in entries
        for trigger in entry.get("triggers", [])
    }


def _custom_labels(directory: Path) -> set[str]:
    """Every label !add_photo minted or proposed -- names too, however they got there.

    Read separately from _pack_names because they are a different kind of
    thing. A people.json trigger is curated by the tagger; a custom label is
    free text somebody typed into a group chat, and treating arbitrary free
    text as a forbidden literal is what lets one message wedge this gate
    permanently. See _repo_vocabulary for the half that makes that safe.

    The approval queue's sidecars (``pending/*.json``) count too: a label
    waiting there is the same typed text, one click earlier.
    """
    labels: set[str] = set()
    for entry in _pack_document(directory / "custom.json").values():
        if isinstance(entry, list):
            labels |= {str(label).lstrip("!").strip() for label in entry}
    for sidecar in sorted((directory / "pending").glob("*.json")):
        proposed = _pack_document(sidecar).get("labels")
        if isinstance(proposed, list):
            labels |= {str(label).lstrip("!").strip() for label in proposed}
    return labels


def _pack_document(path: Path) -> dict[str, Any]:
    """One album manifest, or an empty mapping when it is absent or broken."""
    if not path.is_file():
        return {}
    try:
        document = json.loads(path.read_text(encoding="utf-8"))
    except OSError, json.JSONDecodeError:
        return {}
    return document if isinstance(document, dict) else {}


def _database_names(path: Path) -> set[str]:
    """Display names of everybody the bot has ever seen speak."""
    if not path.is_file():
        return set()
    try:
        connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
    except sqlite3.Error:
        return set()
    # Literal statements, not a table name interpolated into one: a
    # formatted query is a SQL-injection shape even when every input is a
    # constant, and this repository takes no lint suppressions.
    queries = (
        "SELECT DISTINCT display_name FROM message_counts",
        "SELECT DISTINCT display_name FROM group_people",
        "SELECT DISTINCT display_name FROM message_events",
    )
    found: set[str] = set()
    with connection:
        for query in queries:
            try:
                rows = connection.execute(query)
            except sqlite3.Error:
                continue
            found.update(str(row[0]) for row in rows if row[0])
    connection.close()
    return found


def _database_groups(path: Path) -> set[str]:
    """Every real group id the bot has seen, in both spellings.

    Both, because they are the same group written two ways: /v1/groups
    reports "group.<base64>" and an inbound envelope carries the id that
    base64 decodes to. A check that knew only the first would have missed the
    one that was sitting in tests/test_config.py.
    """
    if not path.is_file():
        return set()
    try:
        connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
    except sqlite3.Error:
        return set()
    queries = (
        "SELECT DISTINCT group_id FROM group_meta",
        "SELECT DISTINCT group_id FROM message_events",
    )
    found: set[str] = set()
    with connection:
        for query in queries:
            try:
                rows = connection.execute(query)
            except sqlite3.Error:
                continue
            found.update(str(row[0]) for row in rows if row[0])
    connection.close()
    inner: set[str] = set()
    for group in found:
        payload = group.removeprefix("group.")
        # The leading run as well as the whole id: a log line quoted into a
        # commit message says "group.<first few chars>..." and stops, and
        # matching only the full id let exactly that through once.
        inner.add(payload[:PREFIX])
        try:
            inner.add(base64.b64decode(payload, validate=True).decode("utf-8"))
        except ValueError, UnicodeDecodeError:
            continue
    return found | inner


def _env_values(path: Path) -> set[str]:
    """Phone numbers, group ids and inviters from the deployment's .env."""
    if not path.is_file():
        return set()
    interesting = {"PHONE_NUMBER", "GROUP_IDS", "GROUP_ID", "TRUSTED_INVITERS"}
    found: set[str] = set()
    for line in path.read_text(encoding="utf-8").splitlines():
        key, _, value = line.partition("=")
        if key.strip() in interesting:
            found.update(part.strip() for part in value.split(",") if part.strip())
    return found


def forbidden(permitted: set[str]) -> tuple[set[str], list[str]]:
    """Every literal that must not be committed, and where they came from.

    The sources are all outside the repository on purpose. A list of real
    names kept *in* the repository to check against would be the leak it is
    meant to prevent.
    """
    pack = os.environ.get("PHOTO_PACK_DIR", "").strip()
    directory = (
        Path(pack)
        if pack
        else Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
        / "signal-bot"
        / "pack"
    )
    sources: list[str] = []
    words: set[str] = set()
    for label, found in (
        (f"pack manifest ({directory})", _pack_names(directory)),
        (f"pack custom tags ({directory})", _custom_labels(directory) - _repo_vocabulary()),
        ("message database", _database_names(REPO / "signal_bot.sqlite3")),
        ("group ids", _database_groups(REPO / "signal_bot.sqlite3")),
        (".env", _env_values(REPO / ".env")),
    ):
        if found:
            sources.append(label)
            words |= found
    return {
        word.lower() for word in words if len(word) >= MIN_NAME and word.lower() not in permitted
    }, sources


def _repo_vocabulary() -> set[str]:
    """Words this public repository already contains as its own command names.

    Subtracted from the custom labels, and only from those. A word that is
    committed here *already* cannot be a private name this gate is stopping
    from getting out -- it is out. Without the subtraction, one member
    sending "!add_photo kot <photo>" would mint the label "kot", and the gate
    would then find "kot" in aliases.json and refuse every commit from then
    on, fixable only by hand-editing an album the committer cannot see.

    people.json triggers are deliberately *not* filtered this way: those are
    curated real names, and one of them turning up in the repository is
    exactly the leak worth failing on.
    """
    try:
        document = json.loads((REPO / "signal_bot" / "aliases.json").read_text(encoding="utf-8"))
    except OSError, json.JSONDecodeError:
        return set()
    return {
        str(trigger).lstrip("!").strip().lower()
        for name, triggers in document.items()
        if not name.startswith("_") and isinstance(triggers, list)
        for trigger in triggers
    }