kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
37 lines
1.2 KiB
Python
37 lines
1.2 KiB
Python
"""Tests for the embedding wrapper (using the fake model fixture)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
|
|
from wikikb import config, embed
|
|
|
|
|
|
def test_embed_passages_shape_and_normalization(fake_model) -> None:
|
|
vectors = embed.embed_passages(["a", "b", "c"], model=fake_model)
|
|
assert vectors.shape == (3, config.EMBED_DIM)
|
|
norms = np.linalg.norm(vectors, axis=1)
|
|
assert np.allclose(norms, 1.0, atol=1e-5)
|
|
|
|
|
|
def test_embed_passages_empty(fake_model) -> None:
|
|
vectors = embed.embed_passages([], model=fake_model)
|
|
assert vectors.shape == (0, config.EMBED_DIM)
|
|
|
|
|
|
def test_embed_query_normalized(fake_model) -> None:
|
|
vec = embed.embed_query("domain limit", model=fake_model)
|
|
assert vec.shape == (config.EMBED_DIM,)
|
|
assert np.isclose(np.linalg.norm(vec), 1.0, atol=1e-5)
|
|
|
|
|
|
def test_get_model_returns_cached(monkeypatch, fake_model) -> None:
|
|
monkeypatch.setattr(embed, "_model", fake_model)
|
|
assert embed.get_model() is fake_model
|
|
|
|
|
|
def test_normalize_handles_zero_vector() -> None:
|
|
zero = np.zeros((1, config.EMBED_DIM), dtype=np.float32)
|
|
out = embed._normalize(zero)
|
|
assert np.all(out == 0.0) # zero norm replaced by 1, stays zero
|