"""Tests for markdown chunking.""" from __future__ import annotations from wikikb import chunker _DOC = """--- title: "Traits" url: https://ck3.paradoxwikis.com/Traits revid: 1 categories: ["Character"] --- # Traits Intro paragraph. ## Congenital Genius gives +5 to all skills. ## Physique Herculean gives +1 health. """ def test_parse_frontmatter_extracts_meta_and_body() -> None: meta, body = chunker.parse_frontmatter(_DOC) assert meta["title"] == "Traits" assert meta["url"].endswith("/Traits") assert body.lstrip().startswith("# Traits") def test_parse_frontmatter_absent() -> None: meta, body = chunker.parse_frontmatter("no frontmatter here") assert meta == {} assert body == "no frontmatter here" def test_chunk_document_sections() -> None: chunks = chunker.chunk_document(_DOC) sections = {c.section for c in chunks} assert {"Traits", "Congenital", "Physique"} <= sections assert all(c.title == "Traits" for c in chunks) assert any("Genius" in c.text for c in chunks) def test_window_splits_long_sections_with_overlap() -> None: long = "# T\n\n" + ("word " * 2000) chunks = chunker.chunk_document(long, target_chars=200, overlap_chars=50) assert len(chunks) > 1 assert all(len(c.text) <= 200 for c in chunks) def test_window_short_text_single_chunk() -> None: assert chunker._window("short", size=100, overlap=10) == ["short"] def test_parse_frontmatter_ignores_lines_without_colon() -> None: text = '---\ntitle: "X"\nnocolonhere\n---\n\nbody' meta, _ = chunker.parse_frontmatter(text) assert meta == {"title": "X"} def test_split_sections_drops_empty_text_sections() -> None: # "H1" has only blank lines under it -> filtered out; "Overview" and "H2" kept. body = "text0\n# H1\n\n\n# H2\nx" sections = chunker._split_sections(body) headings = [h for h, _ in sections] assert headings == ["Overview", "H2"] def test_split_sections_body_ending_with_heading() -> None: # trailing heading leaves the buffer empty at loop end (no final flush) assert chunker._split_sections("text\n# H") == [("Overview", "text")] def test_chunk_document_skips_whitespace_only_windows() -> None: doc = "# H\n\nA" + " " * 300 + "B" chunks = chunker.chunk_document(doc, target_chars=100, overlap_chars=10) assert chunks # some windows are all-whitespace and are skipped assert all(c.text.strip() for c in chunks)