kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
76 lines
2.8 KiB
Python
76 lines
2.8 KiB
Python
#!/usr/bin/env python
|
|
"""Verify our local mirror matches the LIVE wiki (no corruption / staleness).
|
|
|
|
For each key page it re-fetches the live page, runs it through the SAME
|
|
HTML->markdown pipeline used at scrape time, and checks:
|
|
1. the freshly-converted live body is byte-identical to our stored copy
|
|
(proves faithful capture + no drift since the scrape), and
|
|
2. specific factual tokens (trait names, modifier numbers) appear in BOTH the
|
|
live page and our local copy — printed side by side so "the same" is visible.
|
|
|
|
DONE = every page's key facts match in live & local (byte-identical is reported
|
|
too, and flagged as WARN if the wiki was edited since the scrape).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import difflib
|
|
import sys
|
|
|
|
import httpx
|
|
|
|
from wikikb import config, html2md
|
|
from wikikb.scraper import Throttle, fetch_page, load_manifest, sanitize_filename
|
|
|
|
# page -> factual tokens that must appear identically in live + local
|
|
FACTS: dict[str, list[str]] = {
|
|
"Traits": ["Genius", "Intelligent", "Herculean", "Beautiful", "+5"],
|
|
"Attributes": ["Diplomacy", "Stewardship", "Prowess", "Domain Taxes"],
|
|
"Succession": ["Primogeniture", "Partition"],
|
|
}
|
|
|
|
|
|
def _stored_text(title: str) -> str:
|
|
entry = load_manifest().get(title, {})
|
|
filename = entry.get("filename") or sanitize_filename(title)
|
|
return (config.CORPUS_DIR / filename).read_text(encoding="utf-8")
|
|
|
|
|
|
def _line_with(text: str, token: str) -> str:
|
|
return next((ln.strip() for ln in text.splitlines() if token in ln), "")
|
|
|
|
|
|
def main() -> int:
|
|
ok = True
|
|
with httpx.Client(headers={"User-Agent": config.USER_AGENT}, timeout=30) as client:
|
|
throttle = Throttle()
|
|
for title, facts in FACTS.items():
|
|
page = fetch_page(client, title, throttle=throttle)
|
|
if page is None:
|
|
print(f"[FAIL] {title}: could not fetch live page")
|
|
ok = False
|
|
continue
|
|
stored = _stored_text(page.title)
|
|
fresh = html2md.html_to_markdown(page.html)
|
|
|
|
identical = fresh in stored # stored = frontmatter + "# Title" + this body
|
|
ratio = difflib.SequenceMatcher(None, fresh, stored).ratio()
|
|
tag = "PASS" if identical else "WARN"
|
|
note = "byte-identical to stored copy" if identical else f"differs (ratio {ratio:.3f})"
|
|
print(f"[{tag}] {page.title} (rev {page.revid}): freshly-converted live body {note}")
|
|
|
|
for fact in facts:
|
|
same = fact in fresh and fact in stored
|
|
ok &= same
|
|
print(
|
|
f" [{'PASS' if same else 'FAIL'}] {fact!r} in live & local -> {same}"
|
|
f" | live: {_line_with(fresh, fact)[:80]!r}"
|
|
)
|
|
|
|
print(f"\nFIDELITY VERIFICATION: {'PASS' if ok else 'FAIL'}")
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|