"""Split corpus markdown documents into overlapping, heading-scoped chunks.""" from __future__ import annotations import re from dataclasses import dataclass from . import config _FRONTMATTER = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL) _HEADING = re.compile(r"^#{1,6}\s+(.*\S)\s*$") @dataclass(frozen=True) class Chunk: """A retrievable unit of text with its source page + section.""" title: str url: str source: str variant: str section: str text: str def parse_frontmatter(text: str) -> tuple[dict[str, str], str]: """Return (metadata, body). Metadata values are unquoted single-line scalars.""" match = _FRONTMATTER.match(text) if not match: return {}, text meta: dict[str, str] = {} for line in match.group(1).splitlines(): if ":" in line: key, value = line.split(":", 1) meta[key.strip()] = value.strip().strip('"') return meta, text[match.end() :] def _split_sections(body: str) -> list[tuple[str, str]]: """Group body lines under their most recent heading.""" sections: list[tuple[str, str]] = [] heading = "Overview" buffer: list[str] = [] for line in body.splitlines(): match = _HEADING.match(line) if match: if buffer: sections.append((heading, "\n".join(buffer).strip())) buffer = [] heading = match.group(1) else: buffer.append(line) if buffer: sections.append((heading, "\n".join(buffer).strip())) return [(h, t) for h, t in sections if t] def _window(text: str, size: int, overlap: int) -> list[str]: """Slice text into <=size windows overlapping by `overlap` chars.""" if len(text) <= size: return [text] step = max(size - overlap, 1) return [text[start : start + size] for start in range(0, len(text), step)] def chunk_document( text: str, *, target_chars: int = config.CHUNK_TARGET_CHARS, overlap_chars: int = config.CHUNK_OVERLAP_CHARS, ) -> list[Chunk]: """Parse a corpus document into heading-scoped, size-bounded chunks.""" meta, body = parse_frontmatter(text) title = meta.get("title", "") url = meta.get("url", "") source = meta.get("source", "mediawiki") page_variant = meta.get("variant", "unknown") chunks: list[Chunk] = [] for heading, section_text in _split_sections(body): if heading.lower() in config.NOISE_SECTIONS: continue for piece in _window(section_text, target_chars, overlap_chars): stripped = piece.strip() if stripped: chunks.append( Chunk( title=title, url=url, source=source, variant=page_variant, section=heading, text=stripped, ) ) return chunks