mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-10 19:27:20 +01:00
Krzysztof kuhy Rudnicki
08599e6056
!sztuka posts a work of art, skewed 60% Polish; !wiersz posts one short public-domain Polish poem, whole. Both follow the rule the rest of the bot already keeps: the repository commits identifiers, and the words arrive over the wire when somebody asks. !wiersz was written the other way first, and the gate caught it. A pack of short public-domain poems was harvested, name-filtered and committed, and check_personal_data.sh refused it: three lines carried a real group member's given name, and two still did after a filter that had discarded 31% of the corpus. That is structural, not a tuning problem -- Polish verse capitalises every line, so a line-initial capital carries no signal, and that is exactly where a name sits. The pack now holds Wolne Lektury slugs; the poem, its title and its author are fetched at post time and appear nowhere in git. It is also the better version: with no name filter there are no false positives, so the pool is larger and the attribution is a full name again. The gate then refused the art category list too, over a painter whose given name is somebody's. That is the popes.json case exactly, so it takes the same answer: a file-scoped allowlist entry, still refused everywhere else. The Commons harvest is now three modules by what they are, not by where the line count fell: commons_api is the conversation with Commons, commons_harvest turns categories into a pool of licensed page ids, commons_cache makes an interrupted run resumable. Along the way it gained licences in the same request that lists a category (the second pass was 1,500 requests at 2.3s each), maxlag instead of a guessed sleep, 5xx retries because one 504 used to destroy a whole run, and Wikidata for works by creators of Polish citizenship: the Polish pool grew 15,221 -> 20,092. Batching categories through CirrusSearch looked like a further 6x and was measured out again: 2,404 files where the authoritative enumeration returns 2,429, a different ~1% missing on every run. The offline run is 19s where it was four minutes, almost all of it the deafness gate waiting out three thirty-second liveness probes to prove something provable in three; the tick is configuration now, and e2e enforces a 40s budget on itself so that never silently comes back.
75 lines
2.7 KiB
Python
75 lines
2.7 KiB
Python
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""Reading Wolne Lektury's plain text, which both !wiersz and the harvest do."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from signal_bot.wolnelektury import PUBLIC_DOMAIN, Poem, parse
|
|
|
|
BODY = "Pierwsza linijka wiersza,\ndruga linijka wiersza."
|
|
|
|
|
|
def _book(
|
|
*,
|
|
header: str = "Maria Konopnicka\n\nA co wam śpiewać\n",
|
|
body: str = BODY,
|
|
footer: str = PUBLIC_DOMAIN,
|
|
) -> str:
|
|
return f"{header}\n\n{body}\n\n\n-----\n{footer}\n"
|
|
|
|
|
|
def test_it_reads_the_author_the_title_and_the_poem() -> None:
|
|
poem = parse(_book())
|
|
assert poem == Poem(title="A co wam śpiewać", author="Maria Konopnicka", text=BODY)
|
|
|
|
|
|
def test_the_footer_is_not_part_of_the_poem() -> None:
|
|
# The rule and everything after it is Wolne Lektury's own boilerplate --
|
|
# a licence notice and a funding credit -- and posting it would double
|
|
# the length of every reply.
|
|
poem = parse(_book())
|
|
assert poem is not None
|
|
assert "wolnelektury" not in poem.text.lower()
|
|
assert PUBLIC_DOMAIN not in poem.text
|
|
|
|
|
|
def test_a_work_that_is_not_public_domain_is_refused() -> None:
|
|
"""The corpus is not a licence: it hosts living authors too.
|
|
|
|
Measured on the real catalogue, a third of it came back non-free, so
|
|
this is the check that keeps !wiersz to the public domain rather than a
|
|
hope about where the poems came from.
|
|
"""
|
|
assert parse(_book(footer="Ten utwór jest udostępniony na licencji CC BY-SA.")) is None
|
|
|
|
|
|
def test_a_translation_is_refused() -> None:
|
|
# A poem translated into Polish is not a Polish poem, and the credit
|
|
# line under the title is the only place the text says so.
|
|
header = "Tristan Derème\n\n[Ach! rzucić sieci zdarte…]\ntłum. Stefan Napierski"
|
|
assert parse(_book(header=header)) is None
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["", "jeden\n\ndwa\n", "\n\n\n"])
|
|
def test_anything_not_shaped_like_a_book_is_refused(raw: str) -> None:
|
|
# Skipped rather than guessed at: a guess here gets posted to a group.
|
|
assert parse(f"{raw}\n-----\n{PUBLIC_DOMAIN}\n") is None
|
|
|
|
|
|
@pytest.mark.parametrize("missing", ["author", "title", "body"])
|
|
def test_a_book_missing_a_field_is_refused(missing: str) -> None:
|
|
parts = {"author": "Maria Konopnicka", "title": "A co wam śpiewać", "body": BODY}
|
|
parts[missing] = ""
|
|
assert (
|
|
parse(_book(header=f"{parts['author']}\n\n{parts['title']}\n", body=parts["body"])) is None
|
|
)
|
|
|
|
|
|
def test_blank_lines_are_stanza_breaks_and_not_lines_of_poetry() -> None:
|
|
# A four-stanza poem would otherwise look twice as long as it reads,
|
|
# and the length cap is what decides whether it ships at all.
|
|
poem = parse(_book(body="jeden\n\ndwa\n\ntrzy"))
|
|
assert poem is not None
|
|
assert poem.lines == 3
|