kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
96 lines
2.9 KiB
Python
96 lines
2.9 KiB
Python
"""Split corpus markdown documents into overlapping, heading-scoped chunks."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
from . import config
|
|
|
|
_FRONTMATTER = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL)
|
|
_HEADING = re.compile(r"^#{1,6}\s+(.*\S)\s*$")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Chunk:
|
|
"""A retrievable unit of text with its source page + section."""
|
|
|
|
title: str
|
|
url: str
|
|
source: str
|
|
variant: str
|
|
section: str
|
|
text: str
|
|
|
|
|
|
def parse_frontmatter(text: str) -> tuple[dict[str, str], str]:
|
|
"""Return (metadata, body). Metadata values are unquoted single-line scalars."""
|
|
match = _FRONTMATTER.match(text)
|
|
if not match:
|
|
return {}, text
|
|
meta: dict[str, str] = {}
|
|
for line in match.group(1).splitlines():
|
|
if ":" in line:
|
|
key, value = line.split(":", 1)
|
|
meta[key.strip()] = value.strip().strip('"')
|
|
return meta, text[match.end() :]
|
|
|
|
|
|
def _split_sections(body: str) -> list[tuple[str, str]]:
|
|
"""Group body lines under their most recent heading."""
|
|
sections: list[tuple[str, str]] = []
|
|
heading = "Overview"
|
|
buffer: list[str] = []
|
|
for line in body.splitlines():
|
|
match = _HEADING.match(line)
|
|
if match:
|
|
if buffer:
|
|
sections.append((heading, "\n".join(buffer).strip()))
|
|
buffer = []
|
|
heading = match.group(1)
|
|
else:
|
|
buffer.append(line)
|
|
if buffer:
|
|
sections.append((heading, "\n".join(buffer).strip()))
|
|
return [(h, t) for h, t in sections if t]
|
|
|
|
|
|
def _window(text: str, size: int, overlap: int) -> list[str]:
|
|
"""Slice text into <=size windows overlapping by `overlap` chars."""
|
|
if len(text) <= size:
|
|
return [text]
|
|
step = max(size - overlap, 1)
|
|
return [text[start : start + size] for start in range(0, len(text), step)]
|
|
|
|
|
|
def chunk_document(
|
|
text: str,
|
|
*,
|
|
target_chars: int = config.CHUNK_TARGET_CHARS,
|
|
overlap_chars: int = config.CHUNK_OVERLAP_CHARS,
|
|
) -> list[Chunk]:
|
|
"""Parse a corpus document into heading-scoped, size-bounded chunks."""
|
|
meta, body = parse_frontmatter(text)
|
|
title = meta.get("title", "")
|
|
url = meta.get("url", "")
|
|
source = meta.get("source", "mediawiki")
|
|
page_variant = meta.get("variant", "unknown")
|
|
chunks: list[Chunk] = []
|
|
for heading, section_text in _split_sections(body):
|
|
if heading.lower() in config.NOISE_SECTIONS:
|
|
continue
|
|
for piece in _window(section_text, target_chars, overlap_chars):
|
|
stripped = piece.strip()
|
|
if stripped:
|
|
chunks.append(
|
|
Chunk(
|
|
title=title,
|
|
url=url,
|
|
source=source,
|
|
variant=page_variant,
|
|
section=heading,
|
|
text=stripped,
|
|
)
|
|
)
|
|
return chunks
|