kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
"""Tests for edition/variant tagging on wikis that cover two editions at once."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from wikikb import config, variant
|
|
|
|
# Ordered most-specific-first, exactly as a profile declares it.
|
|
_KCD = (("kcd2", "KCD2"), ("kcd1", "KCD"))
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _variants(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(config, "VARIANTS", _KCD)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("title", "categories", "expected"),
|
|
[
|
|
("Alchemy/KCD2", [], "kcd2"), # title suffix
|
|
("Common longsword", ["KCD2: Weapons"], "kcd2"), # bare title, category only
|
|
("Lockpicking", ["KCD: Skills"], "kcd1"),
|
|
# "KCD main quests" has no colon: a "KCD:"-prefix test would miss it, and a
|
|
# naive "KCD" test would also match every KCD2 category.
|
|
("Awakening", ["KCD main quests"], "kcd1"),
|
|
("Weapons", ["KCD: Items", "KCD2: Weapons"], "both"),
|
|
("Moravia", ["Countries"], "unknown"),
|
|
],
|
|
)
|
|
def test_classify(title: str, categories: list[str], expected: str) -> None:
|
|
assert variant.classify(title, categories) == expected
|
|
|
|
|
|
def test_classify_without_configured_variants(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
# Most wikis cover one subject; tagging is off unless a profile declares it.
|
|
monkeypatch.setattr(config, "VARIANTS", ())
|
|
assert variant.classify("Alchemy/KCD2", ["KCD2: Skills"]) == "unknown"
|
|
|
|
|
|
def test_matches_excludes_only_the_other_edition() -> None:
|
|
assert variant.matches("kcd2", "kcd2")
|
|
assert variant.matches("both", "kcd2")
|
|
assert variant.matches("unknown", "kcd2") # untagged pages stay in scope
|
|
assert not variant.matches("kcd1", "kcd2")
|
|
|
|
|
|
def test_matches_any_scope() -> None:
|
|
for name in ("kcd1", "kcd2", "both", "unknown"):
|
|
assert variant.matches(name, "any")
|