"""Tests for the pluggable page sources and host detection."""
from __future__ import annotations
import httpx
import pytest
from wikikb import config
from wikikb.sources import (
HTML,
MEDIAWIKI,
MediaWikiSource,
SitemapHtmlSource,
detect_source_kind,
)
_SITEMAP = (
"https://w.example/Common+Longsword"
"https://w.example/Stealth"
)
_INDEX = "https://w.example/s1.xml"
_PAGE = '
'
# --------------------------------------------------------------------- MediaWiki
def _mw(pages: list[dict]) -> MediaWikiSource:
return MediaWikiSource(lambda _params: pages.pop(0))
def test_mediawiki_iter_titles_paginates() -> None:
source = _mw(
[
{
"query": {"allpages": [{"title": "A"}, {"title": "B"}]},
"continue": {"apcontinue": "C"},
},
{"query": {"allpages": [{"title": "C"}]}},
]
)
assert list(source.iter_titles()) == ["A", "B", "C"]
def test_mediawiki_iter_titles_respects_limit() -> None:
source = _mw(
[{"query": {"allpages": [{"title": "A"}, {"title": "B"}]}, "continue": {"apcontinue": "C"}}]
)
assert list(source.iter_titles(limit=1)) == ["A"]
def test_mediawiki_fetch_page_carries_metadata() -> None:
source = _mw(
[
{
"parse": {
"title": "Army",
"revid": 7,
"categories": [{"category": "War_stuff"}],
"text": "hi
",
}
}
]
)
page = source.fetch_page("Army")
assert page is not None
assert (page.title, page.revid, page.source) == ("Army", 7, MEDIAWIKI)
assert page.categories == ["War stuff"]
def test_mediawiki_fetch_page_missing() -> None:
assert _mw([{"error": {"code": "missingtitle"}}]).fetch_page("Nope") is None
# --------------------------------------------------------------------- Sitemap/HTML
def _html_source(responses: dict[str, httpx.Response]) -> SitemapHtmlSource:
return SitemapHtmlSource(
lambda url: responses[url], sitemap_url="https://w.example/sitemap.xml"
)
def test_title_from_url_decodes_slug() -> None:
assert SitemapHtmlSource.title_from_url("https://w.example/Common+Longsword") == (
"Common Longsword"
)
assert SitemapHtmlSource.title_from_url("https://w.example/A_B/") == "A B"
assert SitemapHtmlSource.title_from_url("https://w.example/") == ""
def test_sitemap_iter_titles() -> None:
source = _html_source({"https://w.example/sitemap.xml": httpx.Response(200, text=_SITEMAP)})
assert list(source.iter_titles()) == ["Common Longsword", "Stealth"]
def test_sitemap_iter_titles_skips_empty_slug_and_respects_limit() -> None:
xml = "https://w.example/" + _SITEMAP
source = _html_source({"https://w.example/sitemap.xml": httpx.Response(200, text=xml)})
assert list(source.iter_titles(limit=1)) == ["Common Longsword"]
def test_sitemap_follows_a_sitemap_index() -> None:
source = _html_source(
{
"https://w.example/sitemap.xml": httpx.Response(200, text=_INDEX),
"https://w.example/s1.xml": httpx.Response(200, text=_SITEMAP),
}
)
assert list(source.iter_titles()) == ["Common Longsword", "Stealth"]
def test_sitemap_fetch_page_extracts_the_content_container() -> None:
source = _html_source(
{
"https://w.example/sitemap.xml": httpx.Response(200, text=_SITEMAP),
"https://w.example/Stealth": httpx.Response(200, text=_PAGE),
}
)
list(source.iter_titles()) # populates the title -> url map
page = source.fetch_page("Stealth")
assert page is not None
assert page.source == HTML
assert page.revid == 0 # no revision concept without an API
assert "Stab 122" in page.html
assert "menu" not in page.html # nav is outside the content container
def test_sitemap_fetch_page_404_and_unextractable(monkeypatch) -> None:
monkeypatch.setattr(config, "WIKI_HOST", "https://w.example")
source = _html_source(
{
"https://w.example/Gone": httpx.Response(404),
"https://w.example/Empty": httpx.Response(200, text="no container"),
}
)
assert source.fetch_page("Gone") is None # honest 404 from the server
assert source.fetch_page("Empty") is None # no recognisable content container
def test_sitemap_fetch_page_falls_back_to_a_derived_url(monkeypatch) -> None:
monkeypatch.setattr(config, "WIKI_HOST", "https://w.example")
source = _html_source({"https://w.example/Never%20Listed": httpx.Response(200, text=_PAGE)})
assert source.fetch_page("Never Listed") is not None
# --------------------------------------------------------------------- detection
def test_detect_prefers_the_mediawiki_api() -> None:
def get(url: str) -> httpx.Response:
if url.startswith("https://w.example/api.php"):
return httpx.Response(200, json={"query": {"general": {"generator": "MediaWiki 1.43"}}})
return httpx.Response(404)
assert detect_source_kind(get, "https://w.example") == (MEDIAWIKI, "https://w.example/api.php")
def test_detect_finds_the_wikipedia_style_api_path() -> None:
def get(url: str) -> httpx.Response:
if url.startswith("https://w.example/w/api.php"):
return httpx.Response(200, json={"query": {"general": {"generator": "MediaWiki 1.41"}}})
return httpx.Response(404)
assert detect_source_kind(get, "https://w.example")[1] == "https://w.example/w/api.php"
def test_detect_falls_back_to_sitemap_when_there_is_no_api() -> None:
def get(url: str) -> httpx.Response:
if url.endswith("sitemap.xml"):
return httpx.Response(200, text=_SITEMAP)
return httpx.Response(404)
assert detect_source_kind(get, "https://w.example") == (HTML, "https://w.example/sitemap.xml")
def test_detect_tolerates_a_non_json_or_non_mediawiki_api_response() -> None:
# A host that serves HTML at /api.php (or some other JSON API) must not be
# mistaken for MediaWiki; detection falls through to the sitemap.
def get(url: str) -> httpx.Response:
if "api.php" in url:
return httpx.Response(200, text="not an api")
return httpx.Response(200, text=_SITEMAP)
assert detect_source_kind(get, "https://w.example")[0] == HTML
def test_detect_raises_when_neither_is_available() -> None:
with pytest.raises(RuntimeError, match="neither a MediaWiki API nor a sitemap"):
detect_source_kind(lambda _url: httpx.Response(404), "https://w.example")