wiki-kb/tests/test_scraper.py

"""Tests for the MediaWiki scraper (HTTP mocked with respx)."""

from __future__ import annotations

import httpx
import pytest
import respx
from httpx import Response

from wikikb import config, scraper
from wikikb.scraper import Throttle


def _noop(_: float) -> None:
    return None


def _throttle() -> Throttle:
    return Throttle(interval=0.0, sleep=_noop, clock=lambda: 0.0)


def test_throttle_sleeps_then_skips() -> None:
    calls: list[float] = []
    times = iter([0.0, 0.0, 100.0, 100.0])
    throttle = Throttle(interval=10.0, sleep=calls.append, clock=lambda: next(times))
    throttle.wait()  # now=0, last=0 -> delay 10 -> sleep
    throttle.wait()  # now=100, last=0 -> delay negative -> no sleep
    assert calls == [10.0]


def test_sanitize_filename() -> None:
    assert scraper.sanitize_filename("A/B: C?") == "A_B__C.md"
    assert scraper.sanitize_filename("...") == "untitled.md"


def test_is_core_title() -> None:
    assert scraper.is_core_title("Traits")
    assert scraper.is_core_title("Domain limit")
    assert not scraper.is_core_title("CK3AGOT/Dragon")  # subpage
    assert not scraper.is_core_title("LotR: Realms in Exile")  # mod prefix
    assert not scraper.is_core_title("Elder Kings 2 overview")  # mod prefix


@respx.mock
def test_api_get_success() -> None:
    respx.get(config.API_URL).mock(return_value=Response(200, json={"ok": 1}))
    with httpx.Client() as client:
        data = scraper._api_get(client, {"action": "query"}, throttle=_throttle(), sleep=_noop)
    assert data == {"ok": 1}


@respx.mock
def test_api_get_retries_on_503() -> None:
    route = respx.get(config.API_URL).mock(
        side_effect=[Response(503), Response(200, json={"ok": 2})]
    )
    with httpx.Client() as client:
        data = scraper._api_get(client, {}, throttle=_throttle(), sleep=_noop)
    assert data == {"ok": 2}
    assert route.call_count == 2


@respx.mock
def test_api_get_retries_on_maxlag() -> None:
    respx.get(config.API_URL).mock(
        side_effect=[
            Response(200, json={"error": {"code": "maxlag"}}),
            Response(200, json={"ok": 3}),
        ]
    )
    with httpx.Client() as client:
        data = scraper._api_get(client, {}, throttle=_throttle(), sleep=_noop)
    assert data == {"ok": 3}


@respx.mock
def test_api_get_retries_on_bot_challenge() -> None:
    # Cloudflare serves a challenge as 200 + HTML, so status checks pass and only
    # the JSON decode fails. Without this the challenge page reaches the corpus.
    challenge = Response(200, html="<html><title>Just a moment...</title></html>")
    respx.get(config.API_URL).mock(side_effect=[challenge, Response(200, json={"ok": 4})])
    with httpx.Client() as client:
        assert scraper._api_get(client, {}, throttle=_throttle(), sleep=_noop) == {"ok": 4}


@respx.mock
def test_api_get_exhausts_retries_on_persistent_challenge() -> None:
    respx.get(config.API_URL).mock(
        return_value=Response(200, html="<html><title>Just a moment...</title></html>")
    )
    with httpx.Client() as client, pytest.raises(RuntimeError):
        scraper._api_get(client, {}, throttle=_throttle(), sleep=_noop)


@respx.mock
def test_api_get_exhausts_retries() -> None:
    respx.get(config.API_URL).mock(return_value=Response(503))
    with httpx.Client() as client, pytest.raises(RuntimeError):
        scraper._api_get(client, {}, throttle=_throttle(), sleep=_noop)


def test_manifest_roundtrip(tmp_kb) -> None:
    assert scraper.load_manifest() == {}
    scraper.save_manifest({"A": {"revid": 1}})
    assert scraper.load_manifest() == {"A": {"revid": 1}}


@respx.mock
def test_scrape_writes(tmp_kb) -> None:
    respx.get(config.API_URL).mock(
        side_effect=[
            Response(200, json={"query": {"allpages": [{"title": "Army"}]}}),
            Response(
                200,
                json={
                    "parse": {
                        "title": "Army",
                        "revid": 7,
                        "categories": [],
                        "text": "<p>Levies</p>",
                    }
                },
            ),
        ]
    )
    with httpx.Client() as client:
        result = scraper.scrape(client=client, throttle=_throttle(), log=lambda _: None)
    assert result == {"written": 1, "skipped": 0, "missing": 0}
    assert (config.CORPUS_DIR / "Army.md").exists()
    assert "Army" in scraper.load_manifest()


@respx.mock
def test_scrape_skips_existing(tmp_kb) -> None:
    (config.CORPUS_DIR / "Army.md").write_text("x", encoding="utf-8")
    scraper.save_manifest({"Army": {"revid": 1, "filename": "Army.md"}})
    respx.get(config.API_URL).mock(
        return_value=Response(200, json={"query": {"allpages": [{"title": "Army"}]}})
    )
    with httpx.Client() as client:
        result = scraper.scrape(client=client, throttle=_throttle(), log=lambda _: None)
    assert result == {"written": 0, "skipped": 1, "missing": 0}


@respx.mock
def test_scrape_missing(tmp_kb) -> None:
    respx.get(config.API_URL).mock(
        side_effect=[
            Response(200, json={"query": {"allpages": [{"title": "Ghost"}]}}),
            Response(200, json={"error": {"code": "missingtitle"}}),
        ]
    )
    with httpx.Client() as client:
        result = scraper.scrape(client=client, throttle=_throttle(), log=lambda _: None)
    assert result == {"written": 0, "skipped": 0, "missing": 1}


@respx.mock
def test_scrape_periodic_save_and_owns_client(tmp_kb) -> None:
    titles = [{"title": f"P{i}"} for i in range(25)]
    responses = [Response(200, json={"query": {"allpages": titles}})]
    responses += [
        Response(
            200,
            json={
                "parse": {
                    "title": f"P{i}",
                    "revid": i,
                    "categories": [],
                    "text": f"<p>{i}</p>",
                }
            },
        )
        for i in range(25)
    ]
    respx.get(config.API_URL).mock(side_effect=responses)
    result = scraper.scrape(throttle=_throttle(), log=lambda _: None)  # client=None
    assert result["written"] == 25
    assert len(scraper.load_manifest()) == 25


def test_main_delegates(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(scraper, "scrape", lambda **_: {"written": 1, "skipped": 0, "missing": 0})
    assert scraper.main(["--limit", "5"]) == 0


def test_build_source_picks_mediawiki_by_default() -> None:
    with httpx.Client() as client:
        source = scraper.build_source(client, throttle=_throttle())
    assert source.name == "mediawiki"


def test_build_source_picks_html_when_configured(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(config, "SOURCE_KIND", "html")
    with httpx.Client() as client:
        source = scraper.build_source(client, throttle=_throttle())
    assert source.name == "html"


@respx.mock
def test_get_returns_404_without_retrying() -> None:
    # For the sitemap source a missing page is a normal outcome, not a transient
    # failure, so 404 must come back rather than burn the retry budget.
    route = respx.get("https://w.example/Gone").mock(return_value=Response(404))
    with httpx.Client() as client:
        assert (
            scraper._get(client, "https://w.example/Gone", throttle=_throttle()).status_code == 404
        )
    assert route.call_count == 1


@respx.mock
def test_get_retries_then_succeeds() -> None:
    respx.get("https://w.example/P").mock(side_effect=[Response(503), Response(200, text="ok")])
    with httpx.Client() as client:
        got = scraper._get(client, "https://w.example/P", throttle=_throttle(), sleep=_noop)
    assert got.text == "ok"


@respx.mock
def test_get_exhausts_retries() -> None:
    respx.get("https://w.example/P").mock(return_value=Response(503))
    with httpx.Client() as client, pytest.raises(RuntimeError):
        scraper._get(client, "https://w.example/P", throttle=_throttle(), sleep=_noop)


@respx.mock
def test_scrape_skips_noise_titles(tmp_kb) -> None:
    # Filtering moved out of the source and into scrape(), so it applies identically
    # whichever source enumerated the titles.
    respx.get(config.API_URL).mock(
        side_effect=[
            Response(
                200,
                json={"query": {"allpages": [{"title": "CK3AGOT/Dragon"}, {"title": "Army"}]}},
            ),
            Response(
                200,
                json={"parse": {"title": "Army", "revid": 1, "categories": [], "text": "<p>x</p>"}},
            ),
        ]
    )
    with httpx.Client() as client:
        result = scraper.scrape(client=client, throttle=_throttle(), log=lambda _: None)
    assert result == {"written": 1, "skipped": 0, "missing": 0}
    assert sorted(scraper.load_manifest()) == ["Army"]


def test_is_core_title_subpage_policy_reject(monkeypatch: pytest.MonkeyPatch) -> None:
    # CK3-style wiki: subpages are total-conversion mods.
    monkeypatch.setattr(config, "SUBPAGE_POLICY", "reject")
    monkeypatch.setattr(config, "NOISE_SUBPAGE_TAILS", ())
    assert scraper.is_core_title("Traits")
    assert not scraper.is_core_title("CK3AGOT/Dragon")


def test_is_core_title_subpage_policy_keep(monkeypatch: pytest.MonkeyPatch) -> None:
    # KCD-style wiki: subpages ARE the sequel's pages. Rejecting them would delete
    # 441 pages of exactly the content the user wants.
    monkeypatch.setattr(config, "SUBPAGE_POLICY", "keep")
    monkeypatch.setattr(config, "NOISE_SUBPAGE_TAILS", ("loc",))
    monkeypatch.setattr(config, "MOD_TITLE_PREFIXES", ())
    assert scraper.is_core_title("Alchemy/KCD2")
    assert scraper.is_core_title("Archery master/Uzhitz")
    assert not scraper.is_core_title("Argonautica/loc")


@respx.mock
def test_scrape_with_explicit_titles_skips_enumeration(tmp_kb) -> None:
    # Only the parse call is mocked: enumerating the whole wiki would hit an
    # unmocked allpages request and fail the test.
    respx.get(config.API_URL).mock(
        return_value=Response(
            200,
            json={"parse": {"title": "Army", "revid": 3, "categories": [], "text": "<p>x</p>"}},
        )
    )
    with httpx.Client() as client:
        result = scraper.scrape(
            titles=["Army"], client=client, throttle=_throttle(), log=lambda _: None
        )
    assert result == {"written": 1, "skipped": 0, "missing": 0}


def test_main_parses_titles(monkeypatch: pytest.MonkeyPatch) -> None:
    seen: dict[str, object] = {}

    def fake_scrape(**kwargs: object) -> dict[str, int]:
        seen.update(kwargs)
        return {"written": 0, "skipped": 0, "missing": 0}

    monkeypatch.setattr(scraper, "scrape", fake_scrape)
    assert scraper.main(["--titles", "Alchemy/KCD2, Lockpicking ,"]) == 0
    assert seen["titles"] == ["Alchemy/KCD2", "Lockpicking"]