wiki-kb/tests/test_sources.py

"""Tests for the pluggable page sources and host detection."""

from __future__ import annotations

import httpx
import pytest

from wikikb import config
from wikikb.sources import (
    HTML,
    MEDIAWIKI,
    MediaWikiSource,
    SitemapHtmlSource,
    detect_source_kind,
)

_SITEMAP = (
    "<urlset><url><loc>https://w.example/Common+Longsword</loc></url>"
    "<url><loc>https://w.example/Stealth</loc></url></urlset>"
)
_INDEX = "<sitemapindex><sitemap><loc>https://w.example/s1.xml</loc></sitemap></sitemapindex>"
_PAGE = '<html><body><nav>menu</nav><div id="main-content"><p>Stab 122</p></div></body></html>'


# --------------------------------------------------------------------- MediaWiki


def _mw(pages: list[dict]) -> MediaWikiSource:
    return MediaWikiSource(lambda _params: pages.pop(0))


def test_mediawiki_iter_titles_paginates() -> None:
    source = _mw(
        [
            {
                "query": {"allpages": [{"title": "A"}, {"title": "B"}]},
                "continue": {"apcontinue": "C"},
            },
            {"query": {"allpages": [{"title": "C"}]}},
        ]
    )
    assert list(source.iter_titles()) == ["A", "B", "C"]


def test_mediawiki_iter_titles_respects_limit() -> None:
    source = _mw(
        [{"query": {"allpages": [{"title": "A"}, {"title": "B"}]}, "continue": {"apcontinue": "C"}}]
    )
    assert list(source.iter_titles(limit=1)) == ["A"]


def test_mediawiki_fetch_page_carries_metadata() -> None:
    source = _mw(
        [
            {
                "parse": {
                    "title": "Army",
                    "revid": 7,
                    "categories": [{"category": "War_stuff"}],
                    "text": "<p>hi</p>",
                }
            }
        ]
    )
    page = source.fetch_page("Army")
    assert page is not None
    assert (page.title, page.revid, page.source) == ("Army", 7, MEDIAWIKI)
    assert page.categories == ["War stuff"]


def test_mediawiki_fetch_page_missing() -> None:
    assert _mw([{"error": {"code": "missingtitle"}}]).fetch_page("Nope") is None


# --------------------------------------------------------------------- Sitemap/HTML


def _html_source(responses: dict[str, httpx.Response]) -> SitemapHtmlSource:
    return SitemapHtmlSource(
        lambda url: responses[url], sitemap_url="https://w.example/sitemap.xml"
    )


def test_title_from_url_decodes_slug() -> None:
    assert SitemapHtmlSource.title_from_url("https://w.example/Common+Longsword") == (
        "Common Longsword"
    )
    assert SitemapHtmlSource.title_from_url("https://w.example/A_B/") == "A B"
    assert SitemapHtmlSource.title_from_url("https://w.example/") == ""


def test_sitemap_iter_titles() -> None:
    source = _html_source({"https://w.example/sitemap.xml": httpx.Response(200, text=_SITEMAP)})
    assert list(source.iter_titles()) == ["Common Longsword", "Stealth"]


def test_sitemap_iter_titles_skips_empty_slug_and_respects_limit() -> None:
    xml = "<urlset><url><loc>https://w.example/</loc></url>" + _SITEMAP
    source = _html_source({"https://w.example/sitemap.xml": httpx.Response(200, text=xml)})
    assert list(source.iter_titles(limit=1)) == ["Common Longsword"]


def test_sitemap_follows_a_sitemap_index() -> None:
    source = _html_source(
        {
            "https://w.example/sitemap.xml": httpx.Response(200, text=_INDEX),
            "https://w.example/s1.xml": httpx.Response(200, text=_SITEMAP),
        }
    )
    assert list(source.iter_titles()) == ["Common Longsword", "Stealth"]


def test_sitemap_fetch_page_extracts_the_content_container() -> None:
    source = _html_source(
        {
            "https://w.example/sitemap.xml": httpx.Response(200, text=_SITEMAP),
            "https://w.example/Stealth": httpx.Response(200, text=_PAGE),
        }
    )
    list(source.iter_titles())  # populates the title -> url map
    page = source.fetch_page("Stealth")
    assert page is not None
    assert page.source == HTML
    assert page.revid == 0  # no revision concept without an API
    assert "Stab 122" in page.html
    assert "menu" not in page.html  # nav is outside the content container


def test_sitemap_fetch_page_404_and_unextractable(monkeypatch) -> None:
    monkeypatch.setattr(config, "WIKI_HOST", "https://w.example")
    source = _html_source(
        {
            "https://w.example/Gone": httpx.Response(404),
            "https://w.example/Empty": httpx.Response(200, text="<html><body>no container</body>"),
        }
    )
    assert source.fetch_page("Gone") is None  # honest 404 from the server
    assert source.fetch_page("Empty") is None  # no recognisable content container


def test_sitemap_fetch_page_falls_back_to_a_derived_url(monkeypatch) -> None:
    monkeypatch.setattr(config, "WIKI_HOST", "https://w.example")
    source = _html_source({"https://w.example/Never%20Listed": httpx.Response(200, text=_PAGE)})
    assert source.fetch_page("Never Listed") is not None


# --------------------------------------------------------------------- detection


def test_detect_prefers_the_mediawiki_api() -> None:
    def get(url: str) -> httpx.Response:
        if url.startswith("https://w.example/api.php"):
            return httpx.Response(200, json={"query": {"general": {"generator": "MediaWiki 1.43"}}})
        return httpx.Response(404)

    assert detect_source_kind(get, "https://w.example") == (MEDIAWIKI, "https://w.example/api.php")


def test_detect_finds_the_wikipedia_style_api_path() -> None:
    def get(url: str) -> httpx.Response:
        if url.startswith("https://w.example/w/api.php"):
            return httpx.Response(200, json={"query": {"general": {"generator": "MediaWiki 1.41"}}})
        return httpx.Response(404)

    assert detect_source_kind(get, "https://w.example")[1] == "https://w.example/w/api.php"


def test_detect_falls_back_to_sitemap_when_there_is_no_api() -> None:
    def get(url: str) -> httpx.Response:
        if url.endswith("sitemap.xml"):
            return httpx.Response(200, text=_SITEMAP)
        return httpx.Response(404)

    assert detect_source_kind(get, "https://w.example") == (HTML, "https://w.example/sitemap.xml")


def test_detect_tolerates_a_non_json_or_non_mediawiki_api_response() -> None:
    # A host that serves HTML at /api.php (or some other JSON API) must not be
    # mistaken for MediaWiki; detection falls through to the sitemap.
    def get(url: str) -> httpx.Response:
        if "api.php" in url:
            return httpx.Response(200, text="<html>not an api</html>")
        return httpx.Response(200, text=_SITEMAP)

    assert detect_source_kind(get, "https://w.example")[0] == HTML


def test_detect_raises_when_neither_is_available() -> None:
    with pytest.raises(RuntimeError, match="neither a MediaWiki API nor a sitemap"):
        detect_source_kind(lambda _url: httpx.Response(404), "https://w.example")