kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
77 lines
3.1 KiB
Python
77 lines
3.1 KiB
Python
#!/usr/bin/env python
|
|
"""Probe a wiki host and report how (or whether) it can be mirrored.
|
|
|
|
Phase 0 of the build, as a command:
|
|
|
|
python scripts/detect_wiki.py https://some.wiki.example.com
|
|
|
|
Reports the source kind, the endpoint, the article URL pattern, and — for
|
|
MediaWiki — the engine version and page count. Exits non-zero if the host cannot
|
|
be mirrored at all, so it can gate a scaffold.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
|
|
import httpx
|
|
|
|
from wikikb import config
|
|
from wikikb.sources import MEDIAWIKI, SitemapHtmlSource, detect_source_kind
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = sys.argv[1:] if argv is None else argv
|
|
if not args:
|
|
print("usage: detect_wiki.py <https://host>", file=sys.stderr)
|
|
return 2
|
|
host = args[0].rstrip("/")
|
|
|
|
with httpx.Client(
|
|
headers={"User-Agent": config.USER_AGENT}, timeout=30, follow_redirects=True
|
|
) as client:
|
|
try:
|
|
kind, endpoint = detect_source_kind(client.get, host)
|
|
except RuntimeError as exc:
|
|
print(f"[FAIL] {exc}")
|
|
return 1
|
|
|
|
print(f"[ OK ] source kind : {kind}")
|
|
print(f" endpoint : {endpoint}")
|
|
|
|
if kind == MEDIAWIKI:
|
|
info = client.get(
|
|
f"{endpoint}?action=query&meta=siteinfo&siprop=general|statistics&format=json"
|
|
).json()["query"]
|
|
general, stats = info["general"], info["statistics"]
|
|
base = general.get("base", "")
|
|
pattern = "{host}/wiki/{title}" if "/wiki/" in base else "{host}/{title}"
|
|
print(f" engine : {general.get('generator')}")
|
|
print(f" content : {stats.get('articles')} articles")
|
|
print(f" article URL : {pattern}")
|
|
print(f" licence : {info.get('rightsinfo', {}).get('text', 'unknown')}")
|
|
print("\nScaffold with:")
|
|
print(f" WIKI_KB_HOST={host} WIKI_KB_API_URL={endpoint}")
|
|
if pattern.endswith("/wiki/{title}"):
|
|
print(f" WIKI_KB_PAGE_URL='{host}/wiki/{{title}}'")
|
|
else:
|
|
urls = list(SitemapHtmlSource(client.get, sitemap_url=endpoint).iter_titles())
|
|
print(f" content : {len(urls)} sitemap URLs")
|
|
print(f" sample : {urls[:5]}")
|
|
print("\n[WARN] No API. Revision ids and maxlag are unavailable, so fidelity")
|
|
print(" checks fall back to content diffing and rate limiting is purely")
|
|
print(" self-imposed. Check the site's robots.txt and terms before use.")
|
|
print("\nScaffold with:")
|
|
print(f" WIKI_KB_HOST={host} WIKI_KB_SOURCE=html WIKI_KB_SITEMAP={endpoint}")
|
|
|
|
robots = client.get(f"{host}/robots.txt")
|
|
if robots.status_code == 200 and "gptbot" in robots.text.lower():
|
|
print("\n[NOTE] robots.txt names an AI crawler (e.g. GPTBot). It does not name")
|
|
print(" this tool, but it is the operators' clearest statement about AI")
|
|
print(" ingestion of their content. Your call — read it before scraping.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|