kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
64 lines
1.9 KiB
Python
64 lines
1.9 KiB
Python
"""Tests for the MCP server tool functions."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from wikikb import config, scraper, search, server
|
|
|
|
|
|
def test_wiki_search_maps_results(monkeypatch) -> None:
|
|
monkeypatch.setattr(
|
|
server,
|
|
"search",
|
|
lambda _q, k, variant: [search.Result("T", "u", "mediawiki", "kcd2", "S", "body", 1.0)],
|
|
)
|
|
assert server.wiki_search("q", k=3) == [
|
|
{
|
|
"title": "T",
|
|
"source": "mediawiki",
|
|
"variant": "kcd2",
|
|
"section": "S",
|
|
"url": "u",
|
|
"text": "body",
|
|
}
|
|
]
|
|
|
|
|
|
def test_wiki_get_page_from_manifest(tmp_kb) -> None:
|
|
(config.CORPUS_DIR / "Army.md").write_text("PAGE BODY", encoding="utf-8")
|
|
scraper.save_manifest({"Army": {"filename": "Army.md"}})
|
|
assert "PAGE BODY" in server.wiki_get_page("Army")
|
|
|
|
|
|
def test_wiki_get_page_sanitize_fallback(tmp_kb) -> None:
|
|
(config.CORPUS_DIR / "Foo_Bar.md").write_text("FB", encoding="utf-8")
|
|
assert "FB" in server.wiki_get_page("Foo Bar") # no manifest entry -> sanitized name
|
|
|
|
|
|
def test_wiki_get_page_missing(tmp_kb) -> None:
|
|
assert "No mirrored page" in server.wiki_get_page("Nope")
|
|
|
|
|
|
def test_wiki_list_pages(tmp_kb) -> None:
|
|
scraper.save_manifest({"Army": {}, "Archery": {}, "Barony": {}})
|
|
assert server.wiki_list_pages() == ["Archery", "Army", "Barony"]
|
|
assert server.wiki_list_pages(prefix="ar") == ["Archery", "Army"]
|
|
assert server.wiki_list_pages(limit=1) == ["Archery"]
|
|
|
|
|
|
class _FakeServer:
|
|
def __init__(self) -> None:
|
|
self.registered: list[str] = []
|
|
|
|
def tool(self):
|
|
def decorator(fn):
|
|
self.registered.append(fn.__name__)
|
|
return fn
|
|
|
|
return decorator
|
|
|
|
|
|
def test_build_server_registers_tools() -> None:
|
|
fake = _FakeServer()
|
|
assert server.build_server(fake) is fake
|
|
assert fake.registered == ["wiki_search", "wiki_get_page", "wiki_list_pages"]
|