"""Central configuration: endpoints, paths, crawl politeness, model choice.""" from __future__ import annotations import os from pathlib import Path # --- Wiki endpoints --------------------------------------------------------- # Override WIKI_KB_HOST to re-point this KB at a different MediaWiki (scaffolding # rewrites the literal default below for a new wiki). WIKI_HOST = os.environ.get("WIKI_KB_HOST", "https://ck3.paradoxwikis.com").rstrip("/") # The API path varies by install: paradoxwikis / wiki.gg use /api.php, Wikipedia # uses /w/api.php. The article URL pattern varies too: /{Title} vs /wiki/{Title}. # Override both when re-pointing at a wiki that doesn't match the paradoxwikis layout. API_URL = os.environ.get("WIKI_KB_API_URL", f"{WIKI_HOST}/api.php") PAGE_URL_TEMPLATE = os.environ.get("WIKI_KB_PAGE_URL", f"{WIKI_HOST}/{{title}}") # Namespace 0 is the main (content) namespace. CONTENT_NAMESPACE = 0 # Which page source to use: "mediawiki" (an open api.php — always preferred) or # "html" (sitemap.xml + content-container extraction, for wikis with no API at all). # scripts/detect_wiki.py probes a host and reports which applies. SOURCE_KIND = os.environ.get("WIKI_KB_SOURCE", "mediawiki") SITEMAP_URL = os.environ.get("WIKI_KB_SITEMAP", f"{WIKI_HOST}/sitemap.xml") # --- Subpage policy ----------------------------------------------------------- # Whether "A/B" titles are noise or content. This is PER WIKI and must be measured, # never assumed: on the CK3 wiki subpages are total-conversion mods (reject), while on # the KCD wiki they are the sequel's pages -- "Alchemy/KCD2" -- so rejecting them # deletes 441 pages of exactly the content you want (keep). # Counter(t.rsplit("/", 1)[-1] for t in titles if "/" in t) # answers it in one API pass. SUBPAGE_POLICY = os.environ.get("WIKI_KB_SUBPAGES", "reject") # Subpage tails dropped even when the policy is "keep" (e.g. "loc" localisation dumps). NOISE_SUBPAGE_TAILS = tuple( t.strip().lower() for t in os.environ.get("WIKI_KB_NOISE_TAILS", "").split(",") if t.strip() ) # --- Editions / variants ------------------------------------------------------ # Some wikis cover two editions of one subject in a single namespace (a game and its # sequel, two versions of a product). Declare them as "name:PREFIX" pairs, MOST # SPECIFIC FIRST -- ordering is what lets "KCD" mean edition 1 without also matching # "KCD2". A page matches a variant if its title ends "/PREFIX" or a category starts # with PREFIX. Empty (the default) disables variant tagging entirely. VARIANTS = tuple( (part.split(":", 1)[0].strip(), part.split(":", 1)[1].strip()) for part in os.environ.get("WIKI_KB_VARIANTS", "").split(",") if ":" in part ) # Default search scope. "any" returns everything; a variant name excludes the OTHER # variants while always keeping pages tagged "both" or "unknown". DEFAULT_VARIANT = os.environ.get("WIKI_KB_DEFAULT_VARIANT", "any") # Total-conversion mod pages live in ns0 as subpages ("CK3AGOT/Dragon") or under # these top-level prefixes. They are not base-game mechanics and pollute retrieval, # so the scraper skips them (any title containing "/" is also treated as a mod/subpage). _DEFAULT_MOD_PREFIXES = ( "ck3agot", "lotr", "elder kings", "after the end", "the fallen eagle", "fallen eagle", "godherja", "princes of darkness", "rice", "realms in exile", "a game of thrones", "when the world stopped making sense", "more bookmarks", ) # Override per-wiki with WIKI_KB_MOD_PREFIXES (comma-separated). Branchless so the # import line is always exercised regardless of whether the env var is set. MOD_TITLE_PREFIXES = tuple( p.strip().lower() for p in os.environ.get("WIKI_KB_MOD_PREFIXES", ",".join(_DEFAULT_MOD_PREFIXES)).split(",") if p.strip() ) # --- Politeness / crawl ----------------------------------------------------- # Identify the bot honestly with a contact, per wiki etiquette. # MediaWiki API:Etiquette requires a descriptive User-Agent with a real contact. # Set WIKI_KB_CONTACT — the placeholder is deliberately obvious so an unconfigured # run is embarrassing rather than silently anonymous. CONTACT = os.environ.get("WIKI_KB_CONTACT", "UNCONFIGURED-set-WIKI_KB_CONTACT") USER_AGENT = f"wiki-kb/0.1 (personal offline RAG mirror; contact: {CONTACT})" # Tool prefix for the MCP server: _search / _get_page / _list_pages. MCP_NAMESPACE = os.environ.get("WIKI_KB_NAMESPACE", "wiki") # Minimum seconds between requests (single-flight, ~1 req/s). REQUEST_INTERVAL_S = 1.0 # maxlag tells MediaWiki to reject our request if replication lag is high, # so we back off instead of adding load. See MediaWiki API:Etiquette. MAXLAG = 5 REQUEST_TIMEOUT_S = 30.0 MAX_RETRIES = 5 BACKOFF_BASE_S = 2.0 # exponential: BACKOFF_BASE ** attempt # --- Paths ------------------------------------------------------------------ # WIKI_KB_ROOT isolates a KB's generated data (corpus/index/manifest) — lets one # codebase serve multiple wikis by pointing at different roots. Branchless default. _DEFAULT_ROOT = Path(__file__).resolve().parent.parent ROOT = Path(os.environ.get("WIKI_KB_ROOT", str(_DEFAULT_ROOT))) CORPUS_DIR = ROOT / "corpus" INDEX_DIR = ROOT / "index" DB_PATH = INDEX_DIR / "wiki.db" EMBEDDINGS_PATH = INDEX_DIR / "embeddings.npy" MANIFEST_PATH = ROOT / "manifest.json" # --- Chunking --------------------------------------------------------------- CHUNK_TARGET_CHARS = 2000 # ~500 tokens CHUNK_OVERLAP_CHARS = 200 # Section headings excluded from the index (the page is still mirrored in full and # still served by *_get_page). Wikis carry per-page boilerplate sections that are # both expensive to embed and harmful to retrieval — on the KCD wiki, "Other # languages" tables of localized UI names were 7% of all chunks and repeated every # page's title in eight languages. Empty by default; set per wiki after inspecting # the section histogram. Compared case-insensitively. NOISE_SECTIONS = frozenset( s.strip().lower() for s in os.environ.get("WIKI_KB_NOISE_SECTIONS", "").split(",") if s.strip() ) # --- Embeddings ------------------------------------------------------------- EMBED_MODEL = "BAAI/bge-small-en-v1.5" # 384-dim, CPU-friendly via fastembed EMBED_DIM = 384 EMBED_BATCH_SIZE = 32 def _parse_core_spec(spec: str, total: int) -> set[int]: """Parse "0-3,8" into a set of core ids clamped to [0, total).""" cores: set[int] = set() for part in spec.split(","): part = part.strip() if not part: continue if "-" in part: lo, hi = part.split("-", 1) cores.update(range(int(lo), int(hi) + 1)) else: cores.add(int(part)) return {c for c in cores if 0 <= c < total} def _allowed_cores(total: int, reserved: set[int]) -> list[int]: """Cores usable by heavy jobs = all minus reserved (never empty).""" allowed = sorted(set(range(total)) - reserved) return allowed or list(range(total)) # --- CPU isolation ---------------------------------------------------------- # The bottleneck is core *contention* with the foreground game, not thread count. # Keep RESERVED_CORES free for the game and let heavy jobs use every other core # fully. Override WIKI_KB_RESERVED_CORES (e.g. "0-3" or "8,9"); set "" to use all cores. TOTAL_CPUS = os.cpu_count() or 4 RESERVED_CORES = _parse_core_spec(os.environ.get("WIKI_KB_RESERVED_CORES", "0-3"), TOTAL_CPUS) ALLOWED_CORES = _allowed_cores(TOTAL_CPUS, RESERVED_CORES) # GENTLE BY DEFAULT: a running game floats across ALL cores, so heavy background # work drops FPS no matter which cores it "owns" — nice only softens it. Default to # a single-threaded, single-worker build (~1 core) pinned off RESERVED_CORES and # niced. When the game is CLOSED, set WIKI_KB_EMBED_PARALLEL= for a fast build. EMBED_PARALLEL = int(os.environ.get("WIKI_KB_EMBED_PARALLEL", "1")) EMBED_THREADS = int(os.environ.get("WIKI_KB_EMBED_THREADS", "1")) # Lower scheduling priority so the game still wins on any shared core. NICE_INCREMENT = int(os.environ.get("WIKI_KB_NICE", "10")) # --- Retrieval -------------------------------------------------------------- RRF_K = 60 # Reciprocal Rank Fusion constant DEFAULT_TOP_K = 8 CANDIDATE_POOL = 40 # per-retriever candidates before fusion def page_url(title: str) -> str: """Public wiki URL for a page title.""" return PAGE_URL_TEMPLATE.format(title=title.replace(" ", "_"))