"""Tests for the pluggable page sources and host detection.""" from __future__ import annotations import httpx import pytest from wikikb import config from wikikb.sources import ( HTML, MEDIAWIKI, MediaWikiSource, SitemapHtmlSource, detect_source_kind, ) _SITEMAP = ( "https://w.example/Common+Longsword" "https://w.example/Stealth" ) _INDEX = "https://w.example/s1.xml" _PAGE = '

Stab 122

' # --------------------------------------------------------------------- MediaWiki def _mw(pages: list[dict]) -> MediaWikiSource: return MediaWikiSource(lambda _params: pages.pop(0)) def test_mediawiki_iter_titles_paginates() -> None: source = _mw( [ { "query": {"allpages": [{"title": "A"}, {"title": "B"}]}, "continue": {"apcontinue": "C"}, }, {"query": {"allpages": [{"title": "C"}]}}, ] ) assert list(source.iter_titles()) == ["A", "B", "C"] def test_mediawiki_iter_titles_respects_limit() -> None: source = _mw( [{"query": {"allpages": [{"title": "A"}, {"title": "B"}]}, "continue": {"apcontinue": "C"}}] ) assert list(source.iter_titles(limit=1)) == ["A"] def test_mediawiki_fetch_page_carries_metadata() -> None: source = _mw( [ { "parse": { "title": "Army", "revid": 7, "categories": [{"category": "War_stuff"}], "text": "

hi

", } } ] ) page = source.fetch_page("Army") assert page is not None assert (page.title, page.revid, page.source) == ("Army", 7, MEDIAWIKI) assert page.categories == ["War stuff"] def test_mediawiki_fetch_page_missing() -> None: assert _mw([{"error": {"code": "missingtitle"}}]).fetch_page("Nope") is None # --------------------------------------------------------------------- Sitemap/HTML def _html_source(responses: dict[str, httpx.Response]) -> SitemapHtmlSource: return SitemapHtmlSource( lambda url: responses[url], sitemap_url="https://w.example/sitemap.xml" ) def test_title_from_url_decodes_slug() -> None: assert SitemapHtmlSource.title_from_url("https://w.example/Common+Longsword") == ( "Common Longsword" ) assert SitemapHtmlSource.title_from_url("https://w.example/A_B/") == "A B" assert SitemapHtmlSource.title_from_url("https://w.example/") == "" def test_sitemap_iter_titles() -> None: source = _html_source({"https://w.example/sitemap.xml": httpx.Response(200, text=_SITEMAP)}) assert list(source.iter_titles()) == ["Common Longsword", "Stealth"] def test_sitemap_iter_titles_skips_empty_slug_and_respects_limit() -> None: xml = "https://w.example/" + _SITEMAP source = _html_source({"https://w.example/sitemap.xml": httpx.Response(200, text=xml)}) assert list(source.iter_titles(limit=1)) == ["Common Longsword"] def test_sitemap_follows_a_sitemap_index() -> None: source = _html_source( { "https://w.example/sitemap.xml": httpx.Response(200, text=_INDEX), "https://w.example/s1.xml": httpx.Response(200, text=_SITEMAP), } ) assert list(source.iter_titles()) == ["Common Longsword", "Stealth"] def test_sitemap_fetch_page_extracts_the_content_container() -> None: source = _html_source( { "https://w.example/sitemap.xml": httpx.Response(200, text=_SITEMAP), "https://w.example/Stealth": httpx.Response(200, text=_PAGE), } ) list(source.iter_titles()) # populates the title -> url map page = source.fetch_page("Stealth") assert page is not None assert page.source == HTML assert page.revid == 0 # no revision concept without an API assert "Stab 122" in page.html assert "menu" not in page.html # nav is outside the content container def test_sitemap_fetch_page_404_and_unextractable(monkeypatch) -> None: monkeypatch.setattr(config, "WIKI_HOST", "https://w.example") source = _html_source( { "https://w.example/Gone": httpx.Response(404), "https://w.example/Empty": httpx.Response(200, text="no container"), } ) assert source.fetch_page("Gone") is None # honest 404 from the server assert source.fetch_page("Empty") is None # no recognisable content container def test_sitemap_fetch_page_falls_back_to_a_derived_url(monkeypatch) -> None: monkeypatch.setattr(config, "WIKI_HOST", "https://w.example") source = _html_source({"https://w.example/Never%20Listed": httpx.Response(200, text=_PAGE)}) assert source.fetch_page("Never Listed") is not None # --------------------------------------------------------------------- detection def test_detect_prefers_the_mediawiki_api() -> None: def get(url: str) -> httpx.Response: if url.startswith("https://w.example/api.php"): return httpx.Response(200, json={"query": {"general": {"generator": "MediaWiki 1.43"}}}) return httpx.Response(404) assert detect_source_kind(get, "https://w.example") == (MEDIAWIKI, "https://w.example/api.php") def test_detect_finds_the_wikipedia_style_api_path() -> None: def get(url: str) -> httpx.Response: if url.startswith("https://w.example/w/api.php"): return httpx.Response(200, json={"query": {"general": {"generator": "MediaWiki 1.41"}}}) return httpx.Response(404) assert detect_source_kind(get, "https://w.example")[1] == "https://w.example/w/api.php" def test_detect_falls_back_to_sitemap_when_there_is_no_api() -> None: def get(url: str) -> httpx.Response: if url.endswith("sitemap.xml"): return httpx.Response(200, text=_SITEMAP) return httpx.Response(404) assert detect_source_kind(get, "https://w.example") == (HTML, "https://w.example/sitemap.xml") def test_detect_tolerates_a_non_json_or_non_mediawiki_api_response() -> None: # A host that serves HTML at /api.php (or some other JSON API) must not be # mistaken for MediaWiki; detection falls through to the sitemap. def get(url: str) -> httpx.Response: if "api.php" in url: return httpx.Response(200, text="not an api") return httpx.Response(200, text=_SITEMAP) assert detect_source_kind(get, "https://w.example")[0] == HTML def test_detect_raises_when_neither_is_available() -> None: with pytest.raises(RuntimeError, match="neither a MediaWiki API nor a sitemap"): detect_source_kind(lambda _url: httpx.Response(404), "https://w.example")