kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
48 lines
1.5 KiB
Python
48 lines
1.5 KiB
Python
"""Shared fixtures: temp KB paths and a deterministic fake embedding model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from collections.abc import Iterable
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from wikikb import config
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_kb(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
|
"""Point all config paths at an isolated temp tree."""
|
|
corpus = tmp_path / "corpus"
|
|
index = tmp_path / "index"
|
|
corpus.mkdir()
|
|
index.mkdir()
|
|
monkeypatch.setattr(config, "CORPUS_DIR", corpus)
|
|
monkeypatch.setattr(config, "INDEX_DIR", index)
|
|
monkeypatch.setattr(config, "DB_PATH", index / "wiki.db")
|
|
monkeypatch.setattr(config, "EMBEDDINGS_PATH", index / "embeddings.npy")
|
|
monkeypatch.setattr(config, "MANIFEST_PATH", tmp_path / "manifest.json")
|
|
return tmp_path
|
|
|
|
|
|
class FakeEmbedding:
|
|
"""Deterministic, offline stand-in for fastembed's TextEmbedding."""
|
|
|
|
def _vec(self, text: str) -> np.ndarray:
|
|
digest = hashlib.sha256(text.encode()).digest()
|
|
raw = (digest * (config.EMBED_DIM // len(digest) + 1))[: config.EMBED_DIM]
|
|
return np.frombuffer(raw, dtype=np.uint8).astype(np.float32)
|
|
|
|
def embed(self, texts: Iterable[str], **_kwargs: object) -> list[np.ndarray]:
|
|
return [self._vec(t) for t in texts]
|
|
|
|
def query_embed(self, texts: Iterable[str], **_kwargs: object) -> list[np.ndarray]:
|
|
return [self._vec(t) for t in texts]
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_model() -> FakeEmbedding:
|
|
return FakeEmbedding()
|