kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
49 lines
949 B
TOML
49 lines
949 B
TOML
[project]
|
|
name = "wiki-kb"
|
|
version = "0.1.0"
|
|
description = "Local searchable mirror of the Crusader Kings III wiki (RAG + MCP)."
|
|
requires-python = ">=3.12,<3.13"
|
|
dependencies = [
|
|
"httpx>=0.27",
|
|
"beautifulsoup4>=4.12",
|
|
"markdownify>=0.13",
|
|
"numpy>=1.26",
|
|
"fastembed>=0.3",
|
|
"mcp>=1.2",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=8.0",
|
|
"pytest-cov>=5.0",
|
|
"respx>=0.21",
|
|
"ruff>=0.6",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[tool.setuptools]
|
|
packages = ["wikikb"]
|
|
|
|
[tool.ruff]
|
|
line-length = 100
|
|
target-version = "py312"
|
|
|
|
[tool.ruff.lint]
|
|
select = ["E", "F", "I", "UP", "B", "SIM", "PTH", "RUF"]
|
|
|
|
[tool.pytest.ini_options]
|
|
addopts = "-q"
|
|
testpaths = ["tests"]
|
|
|
|
[tool.coverage.run]
|
|
branch = true
|
|
source = ["wikikb"]
|
|
|
|
[tool.coverage.report]
|
|
show_missing = true
|
|
fail_under = 100
|
|
exclude_lines = ["if __name__ == .__main__.:", "pragma: no cover", "\\.\\.\\."]
|