kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
84 lines
2.4 KiB
Python
84 lines
2.4 KiB
Python
"""Tests for markdown chunking."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from wikikb import chunker
|
|
|
|
_DOC = """---
|
|
title: "Traits"
|
|
url: https://ck3.paradoxwikis.com/Traits
|
|
revid: 1
|
|
categories: ["Character"]
|
|
---
|
|
|
|
# Traits
|
|
|
|
Intro paragraph.
|
|
|
|
## Congenital
|
|
|
|
Genius gives +5 to all skills.
|
|
|
|
## Physique
|
|
|
|
Herculean gives +1 health.
|
|
"""
|
|
|
|
|
|
def test_parse_frontmatter_extracts_meta_and_body() -> None:
|
|
meta, body = chunker.parse_frontmatter(_DOC)
|
|
assert meta["title"] == "Traits"
|
|
assert meta["url"].endswith("/Traits")
|
|
assert body.lstrip().startswith("# Traits")
|
|
|
|
|
|
def test_parse_frontmatter_absent() -> None:
|
|
meta, body = chunker.parse_frontmatter("no frontmatter here")
|
|
assert meta == {}
|
|
assert body == "no frontmatter here"
|
|
|
|
|
|
def test_chunk_document_sections() -> None:
|
|
chunks = chunker.chunk_document(_DOC)
|
|
sections = {c.section for c in chunks}
|
|
assert {"Traits", "Congenital", "Physique"} <= sections
|
|
assert all(c.title == "Traits" for c in chunks)
|
|
assert any("Genius" in c.text for c in chunks)
|
|
|
|
|
|
def test_window_splits_long_sections_with_overlap() -> None:
|
|
long = "# T\n\n" + ("word " * 2000)
|
|
chunks = chunker.chunk_document(long, target_chars=200, overlap_chars=50)
|
|
assert len(chunks) > 1
|
|
assert all(len(c.text) <= 200 for c in chunks)
|
|
|
|
|
|
def test_window_short_text_single_chunk() -> None:
|
|
assert chunker._window("short", size=100, overlap=10) == ["short"]
|
|
|
|
|
|
def test_parse_frontmatter_ignores_lines_without_colon() -> None:
|
|
text = '---\ntitle: "X"\nnocolonhere\n---\n\nbody'
|
|
meta, _ = chunker.parse_frontmatter(text)
|
|
assert meta == {"title": "X"}
|
|
|
|
|
|
def test_split_sections_drops_empty_text_sections() -> None:
|
|
# "H1" has only blank lines under it -> filtered out; "Overview" and "H2" kept.
|
|
body = "text0\n# H1\n\n\n# H2\nx"
|
|
sections = chunker._split_sections(body)
|
|
headings = [h for h, _ in sections]
|
|
assert headings == ["Overview", "H2"]
|
|
|
|
|
|
def test_split_sections_body_ending_with_heading() -> None:
|
|
# trailing heading leaves the buffer empty at loop end (no final flush)
|
|
assert chunker._split_sections("text\n# H") == [("Overview", "text")]
|
|
|
|
|
|
def test_chunk_document_skips_whitespace_only_windows() -> None:
|
|
doc = "# H\n\nA" + " " * 300 + "B"
|
|
chunks = chunker.chunk_document(doc, target_chars=100, overlap_chars=10)
|
|
assert chunks # some windows are all-whitespace and are skipped
|
|
assert all(c.text.strip() for c in chunks)
|