kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
54 lines
1.9 KiB
Python
54 lines
1.9 KiB
Python
"""Tests for CPU isolation and core-spec parsing."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from wikikb import config, cpu
|
|
|
|
|
|
def test_parse_core_spec() -> None:
|
|
assert config._parse_core_spec("0-3", 24) == {0, 1, 2, 3}
|
|
assert config._parse_core_spec("8,10-11", 24) == {8, 10, 11}
|
|
assert config._parse_core_spec("", 24) == set()
|
|
assert config._parse_core_spec("20-100", 24) == {20, 21, 22, 23} # clamped
|
|
|
|
|
|
def test_allowed_cores_excludes_reserved() -> None:
|
|
assert config._allowed_cores(8, {0, 1}) == [2, 3, 4, 5, 6, 7]
|
|
|
|
|
|
def test_allowed_cores_falls_back_when_all_reserved() -> None:
|
|
assert config._allowed_cores(4, {0, 1, 2, 3}) == [0, 1, 2, 3]
|
|
|
|
|
|
def test_isolate_sets_affinity_and_nice(monkeypatch) -> None:
|
|
seen: dict[str, object] = {}
|
|
monkeypatch.setattr(
|
|
os, "sched_setaffinity", lambda pid, cores: seen.__setitem__("aff", (pid, set(cores)))
|
|
)
|
|
monkeypatch.setattr(os, "nice", lambda inc: seen.__setitem__("nice", inc))
|
|
assert cpu.isolate(allowed=[4, 5, 6], nice_increment=7) == [4, 5, 6]
|
|
assert seen["aff"] == (0, {4, 5, 6})
|
|
assert seen["nice"] == 7
|
|
|
|
|
|
def test_isolate_defaults_use_config(monkeypatch) -> None:
|
|
seen: dict[str, object] = {}
|
|
monkeypatch.setattr(
|
|
os, "sched_setaffinity", lambda _pid, cores: seen.__setitem__("c", set(cores))
|
|
)
|
|
monkeypatch.setattr(os, "nice", lambda inc: seen.__setitem__("n", inc))
|
|
assert cpu.isolate() == list(config.ALLOWED_CORES)
|
|
assert seen["c"] == set(config.ALLOWED_CORES)
|
|
assert seen["n"] == config.NICE_INCREMENT
|
|
|
|
|
|
def test_isolate_no_affinity_and_zero_nice(monkeypatch) -> None:
|
|
monkeypatch.delattr(os, "sched_setaffinity", raising=False)
|
|
called = {"nice": False}
|
|
monkeypatch.setattr(os, "nice", lambda _inc: called.__setitem__("nice", True))
|
|
# nice_increment 0 -> nice() skipped; missing sched_setaffinity -> affinity skipped
|
|
assert cpu.isolate(allowed=[0], nice_increment=0) == [0]
|
|
assert called["nice"] is False
|