kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
176 lines
8.3 KiB
Python
176 lines
8.3 KiB
Python
"""Central configuration: endpoints, paths, crawl politeness, model choice."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# --- Wiki endpoints ---------------------------------------------------------
|
|
# Override WIKI_KB_HOST to re-point this KB at a different MediaWiki (scaffolding
|
|
# rewrites the literal default below for a new wiki).
|
|
WIKI_HOST = os.environ.get("WIKI_KB_HOST", "https://ck3.paradoxwikis.com").rstrip("/")
|
|
# The API path varies by install: paradoxwikis / wiki.gg use /api.php, Wikipedia
|
|
# uses /w/api.php. The article URL pattern varies too: /{Title} vs /wiki/{Title}.
|
|
# Override both when re-pointing at a wiki that doesn't match the paradoxwikis layout.
|
|
API_URL = os.environ.get("WIKI_KB_API_URL", f"{WIKI_HOST}/api.php")
|
|
PAGE_URL_TEMPLATE = os.environ.get("WIKI_KB_PAGE_URL", f"{WIKI_HOST}/{{title}}")
|
|
|
|
# Namespace 0 is the main (content) namespace.
|
|
CONTENT_NAMESPACE = 0
|
|
|
|
# Which page source to use: "mediawiki" (an open api.php — always preferred) or
|
|
# "html" (sitemap.xml + content-container extraction, for wikis with no API at all).
|
|
# scripts/detect_wiki.py probes a host and reports which applies.
|
|
SOURCE_KIND = os.environ.get("WIKI_KB_SOURCE", "mediawiki")
|
|
SITEMAP_URL = os.environ.get("WIKI_KB_SITEMAP", f"{WIKI_HOST}/sitemap.xml")
|
|
|
|
# --- Subpage policy -----------------------------------------------------------
|
|
# Whether "A/B" titles are noise or content. This is PER WIKI and must be measured,
|
|
# never assumed: on the CK3 wiki subpages are total-conversion mods (reject), while on
|
|
# the KCD wiki they are the sequel's pages -- "Alchemy/KCD2" -- so rejecting them
|
|
# deletes 441 pages of exactly the content you want (keep).
|
|
# Counter(t.rsplit("/", 1)[-1] for t in titles if "/" in t)
|
|
# answers it in one API pass.
|
|
SUBPAGE_POLICY = os.environ.get("WIKI_KB_SUBPAGES", "reject")
|
|
# Subpage tails dropped even when the policy is "keep" (e.g. "loc" localisation dumps).
|
|
NOISE_SUBPAGE_TAILS = tuple(
|
|
t.strip().lower() for t in os.environ.get("WIKI_KB_NOISE_TAILS", "").split(",") if t.strip()
|
|
)
|
|
|
|
# --- Editions / variants ------------------------------------------------------
|
|
# Some wikis cover two editions of one subject in a single namespace (a game and its
|
|
# sequel, two versions of a product). Declare them as "name:PREFIX" pairs, MOST
|
|
# SPECIFIC FIRST -- ordering is what lets "KCD" mean edition 1 without also matching
|
|
# "KCD2". A page matches a variant if its title ends "/PREFIX" or a category starts
|
|
# with PREFIX. Empty (the default) disables variant tagging entirely.
|
|
VARIANTS = tuple(
|
|
(part.split(":", 1)[0].strip(), part.split(":", 1)[1].strip())
|
|
for part in os.environ.get("WIKI_KB_VARIANTS", "").split(",")
|
|
if ":" in part
|
|
)
|
|
# Default search scope. "any" returns everything; a variant name excludes the OTHER
|
|
# variants while always keeping pages tagged "both" or "unknown".
|
|
DEFAULT_VARIANT = os.environ.get("WIKI_KB_DEFAULT_VARIANT", "any")
|
|
|
|
# Total-conversion mod pages live in ns0 as subpages ("CK3AGOT/Dragon") or under
|
|
# these top-level prefixes. They are not base-game mechanics and pollute retrieval,
|
|
# so the scraper skips them (any title containing "/" is also treated as a mod/subpage).
|
|
_DEFAULT_MOD_PREFIXES = (
|
|
"ck3agot",
|
|
"lotr",
|
|
"elder kings",
|
|
"after the end",
|
|
"the fallen eagle",
|
|
"fallen eagle",
|
|
"godherja",
|
|
"princes of darkness",
|
|
"rice",
|
|
"realms in exile",
|
|
"a game of thrones",
|
|
"when the world stopped making sense",
|
|
"more bookmarks",
|
|
)
|
|
# Override per-wiki with WIKI_KB_MOD_PREFIXES (comma-separated). Branchless so the
|
|
# import line is always exercised regardless of whether the env var is set.
|
|
MOD_TITLE_PREFIXES = tuple(
|
|
p.strip().lower()
|
|
for p in os.environ.get("WIKI_KB_MOD_PREFIXES", ",".join(_DEFAULT_MOD_PREFIXES)).split(",")
|
|
if p.strip()
|
|
)
|
|
|
|
# --- Politeness / crawl -----------------------------------------------------
|
|
# Identify the bot honestly with a contact, per wiki etiquette.
|
|
# MediaWiki API:Etiquette requires a descriptive User-Agent with a real contact.
|
|
# Set WIKI_KB_CONTACT — the placeholder is deliberately obvious so an unconfigured
|
|
# run is embarrassing rather than silently anonymous.
|
|
CONTACT = os.environ.get("WIKI_KB_CONTACT", "UNCONFIGURED-set-WIKI_KB_CONTACT")
|
|
USER_AGENT = f"wiki-kb/0.1 (personal offline RAG mirror; contact: {CONTACT})"
|
|
|
|
# Tool prefix for the MCP server: <namespace>_search / _get_page / _list_pages.
|
|
MCP_NAMESPACE = os.environ.get("WIKI_KB_NAMESPACE", "wiki")
|
|
# Minimum seconds between requests (single-flight, ~1 req/s).
|
|
REQUEST_INTERVAL_S = 1.0
|
|
# maxlag tells MediaWiki to reject our request if replication lag is high,
|
|
# so we back off instead of adding load. See MediaWiki API:Etiquette.
|
|
MAXLAG = 5
|
|
REQUEST_TIMEOUT_S = 30.0
|
|
MAX_RETRIES = 5
|
|
BACKOFF_BASE_S = 2.0 # exponential: BACKOFF_BASE ** attempt
|
|
|
|
# --- Paths ------------------------------------------------------------------
|
|
# WIKI_KB_ROOT isolates a KB's generated data (corpus/index/manifest) — lets one
|
|
# codebase serve multiple wikis by pointing at different roots. Branchless default.
|
|
_DEFAULT_ROOT = Path(__file__).resolve().parent.parent
|
|
ROOT = Path(os.environ.get("WIKI_KB_ROOT", str(_DEFAULT_ROOT)))
|
|
CORPUS_DIR = ROOT / "corpus"
|
|
INDEX_DIR = ROOT / "index"
|
|
DB_PATH = INDEX_DIR / "wiki.db"
|
|
EMBEDDINGS_PATH = INDEX_DIR / "embeddings.npy"
|
|
MANIFEST_PATH = ROOT / "manifest.json"
|
|
|
|
# --- Chunking ---------------------------------------------------------------
|
|
CHUNK_TARGET_CHARS = 2000 # ~500 tokens
|
|
CHUNK_OVERLAP_CHARS = 200
|
|
# Section headings excluded from the index (the page is still mirrored in full and
|
|
# still served by *_get_page). Wikis carry per-page boilerplate sections that are
|
|
# both expensive to embed and harmful to retrieval — on the KCD wiki, "Other
|
|
# languages" tables of localized UI names were 7% of all chunks and repeated every
|
|
# page's title in eight languages. Empty by default; set per wiki after inspecting
|
|
# the section histogram. Compared case-insensitively.
|
|
NOISE_SECTIONS = frozenset(
|
|
s.strip().lower() for s in os.environ.get("WIKI_KB_NOISE_SECTIONS", "").split(",") if s.strip()
|
|
)
|
|
|
|
# --- Embeddings -------------------------------------------------------------
|
|
EMBED_MODEL = "BAAI/bge-small-en-v1.5" # 384-dim, CPU-friendly via fastembed
|
|
EMBED_DIM = 384
|
|
EMBED_BATCH_SIZE = 32
|
|
|
|
|
|
def _parse_core_spec(spec: str, total: int) -> set[int]:
|
|
"""Parse "0-3,8" into a set of core ids clamped to [0, total)."""
|
|
cores: set[int] = set()
|
|
for part in spec.split(","):
|
|
part = part.strip()
|
|
if not part:
|
|
continue
|
|
if "-" in part:
|
|
lo, hi = part.split("-", 1)
|
|
cores.update(range(int(lo), int(hi) + 1))
|
|
else:
|
|
cores.add(int(part))
|
|
return {c for c in cores if 0 <= c < total}
|
|
|
|
|
|
def _allowed_cores(total: int, reserved: set[int]) -> list[int]:
|
|
"""Cores usable by heavy jobs = all minus reserved (never empty)."""
|
|
allowed = sorted(set(range(total)) - reserved)
|
|
return allowed or list(range(total))
|
|
|
|
|
|
# --- CPU isolation ----------------------------------------------------------
|
|
# The bottleneck is core *contention* with the foreground game, not thread count.
|
|
# Keep RESERVED_CORES free for the game and let heavy jobs use every other core
|
|
# fully. Override WIKI_KB_RESERVED_CORES (e.g. "0-3" or "8,9"); set "" to use all cores.
|
|
TOTAL_CPUS = os.cpu_count() or 4
|
|
RESERVED_CORES = _parse_core_spec(os.environ.get("WIKI_KB_RESERVED_CORES", "0-3"), TOTAL_CPUS)
|
|
ALLOWED_CORES = _allowed_cores(TOTAL_CPUS, RESERVED_CORES)
|
|
# GENTLE BY DEFAULT: a running game floats across ALL cores, so heavy background
|
|
# work drops FPS no matter which cores it "owns" — nice only softens it. Default to
|
|
# a single-threaded, single-worker build (~1 core) pinned off RESERVED_CORES and
|
|
# niced. When the game is CLOSED, set WIKI_KB_EMBED_PARALLEL=<n> for a fast build.
|
|
EMBED_PARALLEL = int(os.environ.get("WIKI_KB_EMBED_PARALLEL", "1"))
|
|
EMBED_THREADS = int(os.environ.get("WIKI_KB_EMBED_THREADS", "1"))
|
|
# Lower scheduling priority so the game still wins on any shared core.
|
|
NICE_INCREMENT = int(os.environ.get("WIKI_KB_NICE", "10"))
|
|
|
|
# --- Retrieval --------------------------------------------------------------
|
|
RRF_K = 60 # Reciprocal Rank Fusion constant
|
|
DEFAULT_TOP_K = 8
|
|
CANDIDATE_POOL = 40 # per-retriever candidates before fusion
|
|
|
|
|
|
def page_url(title: str) -> str:
|
|
"""Public wiki URL for a page title."""
|
|
return PAGE_URL_TEMPLATE.format(title=title.replace(" ", "_"))
|