#!/usr/bin/env python """Verify our local mirror matches the LIVE wiki (no corruption / staleness). For each key page it re-fetches the live page, runs it through the SAME HTML->markdown pipeline used at scrape time, and checks: 1. the freshly-converted live body is byte-identical to our stored copy (proves faithful capture + no drift since the scrape), and 2. specific factual tokens (trait names, modifier numbers) appear in BOTH the live page and our local copy — printed side by side so "the same" is visible. DONE = every page's key facts match in live & local (byte-identical is reported too, and flagged as WARN if the wiki was edited since the scrape). """ from __future__ import annotations import difflib import sys import httpx from wikikb import config, html2md from wikikb.scraper import Throttle, fetch_page, load_manifest, sanitize_filename # page -> factual tokens that must appear identically in live + local FACTS: dict[str, list[str]] = { "Traits": ["Genius", "Intelligent", "Herculean", "Beautiful", "+5"], "Attributes": ["Diplomacy", "Stewardship", "Prowess", "Domain Taxes"], "Succession": ["Primogeniture", "Partition"], } def _stored_text(title: str) -> str: entry = load_manifest().get(title, {}) filename = entry.get("filename") or sanitize_filename(title) return (config.CORPUS_DIR / filename).read_text(encoding="utf-8") def _line_with(text: str, token: str) -> str: return next((ln.strip() for ln in text.splitlines() if token in ln), "") def main() -> int: ok = True with httpx.Client(headers={"User-Agent": config.USER_AGENT}, timeout=30) as client: throttle = Throttle() for title, facts in FACTS.items(): page = fetch_page(client, title, throttle=throttle) if page is None: print(f"[FAIL] {title}: could not fetch live page") ok = False continue stored = _stored_text(page.title) fresh = html2md.html_to_markdown(page.html) identical = fresh in stored # stored = frontmatter + "# Title" + this body ratio = difflib.SequenceMatcher(None, fresh, stored).ratio() tag = "PASS" if identical else "WARN" note = "byte-identical to stored copy" if identical else f"differs (ratio {ratio:.3f})" print(f"[{tag}] {page.title} (rev {page.revid}): freshly-converted live body {note}") for fact in facts: same = fact in fresh and fact in stored ok &= same print( f" [{'PASS' if same else 'FAIL'}] {fact!r} in live & local -> {same}" f" | live: {_line_with(fresh, fact)[:80]!r}" ) print(f"\nFIDELITY VERIFICATION: {'PASS' if ok else 'FAIL'}") return 0 if ok else 1 if __name__ == "__main__": sys.exit(main())