From 86565bdef61d54067a1096ca9d7f1806a87f5ebe Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Mon, 10 Aug 2026 15:29:08 +0200 Subject: [PATCH] archwiki_rag: offline semantic search over the Arch Wiki MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indexes the arch-wiki-docs dump that is already on disk and exposes it to the knowledge-rag MCP server, so Arch questions can be asked by symptom ("why doesn't my systemd user unit see DISPLAY under i3") rather than by page title, with no network round-trip at query time. The one Arch-specific MCP server in the wild fetches pages over the network and bundles pacman operations that duplicate the local yay-mcp. This does neither: it only searches, against a local corpus. install.sh installs its own dependencies, wires the MCP config and registers a pacman hook so the index follows package updates. The generated index is machine-specific and regenerable, so it is gitignored rather than committed. _reindex_entry runs under the knowledge-rag interpreter, where mcp_server is importable and this repo's packages are not. Resolving that import with importlib.import_module states the deferral outright instead of a function-level import statement that reads as an oversight, and the backend failure path now re-raises as SystemExit, so the exit-code translation is explicit rather than a swallowed exception. Both replace lint suppressions; the package carries none. pylint runs in its own venv, so beautifulsoup4 and markdownify are declared there too — being in meta/requirements.txt does not make them resolvable to the hook. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YQLwdquDCNmWtvgi3Lk72J --- .pre-commit-config.yaml | 5 + .../archwiki-rag-corpus-2026-07-19.json | 17 ++ .../archwiki-rag-corpus-2026-07-19.json | 69 ++++++++ python_pkg/archwiki_rag/README.md | 81 +++++++++ python_pkg/archwiki_rag/__init__.py | 21 +++ python_pkg/archwiki_rag/__main__.py | 14 ++ python_pkg/archwiki_rag/_reindex_entry.py | 77 +++++++++ python_pkg/archwiki_rag/cli.py | 118 +++++++++++++ python_pkg/archwiki_rag/constants.py | 68 ++++++++ python_pkg/archwiki_rag/convert.py | 142 +++++++++++++++ python_pkg/archwiki_rag/install.sh | 130 ++++++++++++++ python_pkg/archwiki_rag/reindex.py | 131 ++++++++++++++ python_pkg/archwiki_rag/sync.py | 109 ++++++++++++ python_pkg/archwiki_rag/tests/__init__.py | 3 + python_pkg/archwiki_rag/tests/test_cli.py | 161 ++++++++++++++++++ python_pkg/archwiki_rag/tests/test_convert.py | 142 +++++++++++++++ .../archwiki_rag/tests/test_main_entry.py | 22 +++ python_pkg/archwiki_rag/tests/test_reindex.py | 157 +++++++++++++++++ .../archwiki_rag/tests/test_reindex_entry.py | 84 +++++++++ python_pkg/archwiki_rag/tests/test_sync.py | 125 ++++++++++++++ 20 files changed, 1676 insertions(+) create mode 100644 docs/superpowers/contracts/archwiki-rag-corpus-2026-07-19.json create mode 100644 docs/superpowers/evidence/archwiki-rag-corpus-2026-07-19.json create mode 100644 python_pkg/archwiki_rag/README.md create mode 100644 python_pkg/archwiki_rag/__init__.py create mode 100644 python_pkg/archwiki_rag/__main__.py create mode 100644 python_pkg/archwiki_rag/_reindex_entry.py create mode 100644 python_pkg/archwiki_rag/cli.py create mode 100644 python_pkg/archwiki_rag/constants.py create mode 100644 python_pkg/archwiki_rag/convert.py create mode 100755 python_pkg/archwiki_rag/install.sh create mode 100644 python_pkg/archwiki_rag/reindex.py create mode 100644 python_pkg/archwiki_rag/sync.py create mode 100644 python_pkg/archwiki_rag/tests/__init__.py create mode 100644 python_pkg/archwiki_rag/tests/test_cli.py create mode 100644 python_pkg/archwiki_rag/tests/test_convert.py create mode 100644 python_pkg/archwiki_rag/tests/test_main_entry.py create mode 100644 python_pkg/archwiki_rag/tests/test_reindex.py create mode 100644 python_pkg/archwiki_rag/tests/test_reindex_entry.py create mode 100644 python_pkg/archwiki_rag/tests/test_sync.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5a7fc8f..f9fb026a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -204,6 +204,11 @@ repos: - requests - pygame - pillow + # archwiki_rag parses the local arch-wiki-docs HTML dump. pylint runs + # in its own venv, so a package being in meta/requirements.txt is not + # enough for it to resolve the import. + - beautifulsoup4 + - markdownify - gatelock @ git+https://github.com/kuhyx/utils@gatelock-v0.1.0#subdirectory=gatelock - anthropic - rich diff --git a/docs/superpowers/contracts/archwiki-rag-corpus-2026-07-19.json b/docs/superpowers/contracts/archwiki-rag-corpus-2026-07-19.json new file mode 100644 index 00000000..1bd34700 --- /dev/null +++ b/docs/superpowers/contracts/archwiki-rag-corpus-2026-07-19.json @@ -0,0 +1,17 @@ +{ + "title": "archwiki_rag: offline semantic search over the Arch Wiki", + "objective": "Let Claude answer Arch questions by symptom rather than by page title, with no network round-trip, by indexing the arch-wiki-docs dump that is already on disk and exposing it to the knowledge-rag MCP server.", + "acceptance_criteria": [ + "Converts the local arch-wiki-docs dump into a markdown corpus without fetching pages over the network", + "Builds a searchable index and refreshes it via `sync --reindex`", + "install.sh installs its own dependencies, wires the MCP config and registers a pacman hook so the index follows package updates", + "100% branch coverage across every module in python_pkg/archwiki_rag", + "The generated index is not committed (machine-specific, binary, regenerable)" + ], + "out_of_scope": [ + "Duplicating yay-mcp's pacman install/remove/update, PKGBUILD checks or system stats — this package only searches", + "Fetching wiki pages over the network at query time", + "Replacing the generic wiki-kb pipeline for non-Arch wikis" + ], + "verifier": "python -m pytest python_pkg/archwiki_rag --cov=python_pkg.archwiki_rag (61 passed, 100% branch coverage on cli, constants, convert, reindex, sync, _reindex_entry)" +} diff --git a/docs/superpowers/evidence/archwiki-rag-corpus-2026-07-19.json b/docs/superpowers/evidence/archwiki-rag-corpus-2026-07-19.json new file mode 100644 index 00000000..5d72ed4e --- /dev/null +++ b/docs/superpowers/evidence/archwiki-rag-corpus-2026-07-19.json @@ -0,0 +1,69 @@ +{ + "intent": "Give Claude offline semantic search over the whole Arch Wiki, so questions can be asked by symptom (\"systemd user unit can't see DISPLAY under i3\") rather than by page title, with no network round-trip.", + "scope": [ + "python_pkg/archwiki_rag/ — new package: HTML→Markdown converter, sync, reindex, CLI", + "meta/requirements.txt — add markdownify", + "meta/scripts/check_requirements_sync.py — bs4→beautifulsoup4 map, foreign-runtime import exclusion", + "linux_configuration/zsh/claude-tty-fix.zsh — claude-archwiki wrapper", + "Non-goal: any pacman read/write tooling — yay-mcp already covers that", + "Non-goal: non-English wiki trees (~4x corpus, no expected queries)" + ], + "changes": [ + "convert.py renders each page to Markdown with ATX headings, because knowledge-rag chunks .md with a heading-aware splitter and everything else with a blind fixed-width one; HTML is not a supported suffix at all.", + "sync.py writes a page only when the rendered Markdown differs from what is on disk. knowledge-rag's incremental reindex compares mtime+size, not content, and arch-wiki-docs regenerates every file per release — writing unconditionally would re-embed the whole corpus on each upgrade.", + "iter_pages uses rglob: subpages live in nested directories (en/Systemd/User.html). A non-recursive glob silently missed 164 pages, caught by comparing counts against the dump.", + "Corpus lives in its own store (~/.local/share/knowledge-rag-archwiki), not the shared knowledge-rag one; ~27k Arch chunks would dominate BM25 and the embedding space of any other corpus.", + "reindex.py guards on the knowledge-rag instance lock (only taken by its main(), so a direct orchestrator import would bypass it) and on /proc/loadavg load-per-core.", + "Load guard replaced an initial VRAM guard after discovering knowledge-rag ships CPU-only onnxruntime (providers: Azure, CPU — no CUDA), so embedding is CPU-bound and a VRAM check could never fire." + ], + "verification": [ + { + "command": "python -m python_pkg.archwiki_rag sync", + "result": "pass", + "evidence": "converted 2501, changed 164, skipped 8 — the 164 were exactly the newly-discovered subpages; the 8 skips are the md5-named stray non-English pages." + }, + { + "command": "python -m python_pkg.archwiki_rag sync (immediate re-run)", + "result": "pass", + "evidence": "converted 2501, changed 0. Youngest .md mtime age 29.7s, i.e. no file rewritten — the incremental-reindex precondition holds." + }, + { + "command": "python -c 'reindex.blocking_reason(store)' (live, machine under real load)", + "result": "pass", + "evidence": "/proc/loadavg 36.31 over 24 cores = 1.51/core -> 'machine is busy (load 1.51/core); deferring to avoid contention'; ignore_load=True returns None." + }, + { + "command": "ruff check python_pkg/archwiki_rag/ && mypy python_pkg/archwiki_rag/", + "result": "pass", + "evidence": "ruff: All checks passed. mypy: Success, no issues in 15 source files. Zero suppressions added." + }, + { + "command": "pytest python_pkg/archwiki_rag/ --cov --cov-branch", + "result": "pass", + "evidence": "61 passed; 100% statement and branch coverage on all 8 modules." + }, + { + "command": "python meta/scripts/check_requirements_sync.py --package python_pkg", + "result": "pass", + "evidence": "EXIT=0 after adding bs4->beautifulsoup4 and excluding mcp_server as a foreign-runtime import." + }, + { + "command": "end-to-end search_knowledge() against the built index", + "result": "NOT YET RUN", + "evidence": "The initial embedding pass was stopped at ~40% on user instruction (it held ~19 of 24 cores while they were at the machine) and rescheduled to 22:00 via `systemd-run --user --unit=archwiki-rag-index --on-calendar=22:00`. The retrieval-quality acceptance test is therefore still outstanding; the conversion half is verified above." + } + ], + "risks": [ + "Initial embed is CPU-bound and long (~2501 files); the pacman hook detaches it via systemd-run so a pacman transaction is never blocked.", + "The load guard can defer indefinitely on a permanently busy machine; the converted Markdown persists on disk, so the next run resumes with no lost work, and --ignore-load forces it.", + "The pacman hook aborts every transaction if malformed; only Install/Upgrade/Remove Operation values are used, and it must be validated with a real pacman -S before being trusted.", + "claude-extras globs mcp-optional/*.json and so now also loads the archwiki server.", + "knowledge-rag calls _save_metadata() only after the full reindex loop, so a reindex killed mid-run leaves vectors in Chroma with no metadata record; the next run then re-embeds everything AND the orphan cleanup (iterating the empty _indexed_docs) never removes the old chunks, duplicating them. Observed here: 24,225 orphaned embeddings after the interrupted run. Recovery is to delete the store's data/ directory; documents/ is unaffected." + ], + "rollback": [ + "rm /etc/pacman.d/hooks/95-archwiki-rag.hook /usr/local/bin/archwiki-rag-refresh", + "rm -rf ~/.local/share/knowledge-rag-archwiki ~/.claude/mcp-optional/archwiki.json", + "git revert the commit; the shared ~/.local/share/knowledge-rag store is untouched by this change and needs no validation.", + "After rollback, confirm `pacman -S ` still completes — that proves the hook is gone and transactions are unaffected." + ] +} diff --git a/python_pkg/archwiki_rag/README.md b/python_pkg/archwiki_rag/README.md new file mode 100644 index 00000000..80d1c28f --- /dev/null +++ b/python_pkg/archwiki_rag/README.md @@ -0,0 +1,81 @@ +# archwiki_rag + +Offline Arch Wiki corpus for the `knowledge-rag` MCP server. Gives Claude +semantic search across the whole ArchWiki without a network round-trip, so +questions can be asked by _symptom_ rather than by page title: + +> why doesn't my systemd user unit see DISPLAY under i3 + +## Why this exists + +The only Arch-specific MCP server in the wild +([`nihalxkumar/arch-mcp`](https://github.com/nihalxkumar/arch-mcp)) fetches wiki +pages over the network and bundles pacman install/remove/update, PKGBUILD +checks and system stats that duplicate the local `yay-mcp`. This package uses +the `arch-wiki-docs` dump that is already on disk instead, and does nothing but +search. + +## Usage + +```bash +# One-time setup: dependencies, corpus, MCP config, pacman hook. +./install.sh + +# Manual refresh. +PYTHONPATH=~/testsAndMisc python3 -m python_pkg.archwiki_rag sync --reindex + +# Query it. +claude-archwiki +``` + +Then, in the session, `search_knowledge("...")` against the `archwiki` server. + +## Design notes + +Three non-obvious constraints shaped this package. All three come from reading +the `knowledge-rag` source rather than its docs. + +**Markdown, not text.** `knowledge-rag` chunks `.md` files with a heading-aware +splitter and everything else with a blind fixed-width one. ArchWiki pages are +heavily sectioned, so emitting real `##`/`###` headings is what makes chunk +boundaries land on section breaks instead of mid-sentence. HTML is not a +supported suffix at all, so conversion is mandatory regardless. + +**Never rewrite an unchanged file.** The incremental reindex decides a document +is current by comparing stored mtime and size, _not_ content. `arch-wiki-docs` +regenerates all ~2500 files on every release, so writing unconditionally would +bump every mtime and re-embed the entire corpus on each upgrade. `sync.py` +compares rendered output against what is on disk and skips identical pages; +this is the single most load-bearing behaviour in the package, and +`test_sync.py` asserts it directly. + +**Subpages are nested.** `Systemd/User` lives at `en/Systemd/User.html`, not +`en/Systemd_User.html`. A non-recursive glob silently misses 164 pages, many of +them the most useful ones. `iter_pages` uses `rglob` and the documents tree +mirrors the dump's layout. + +## Store isolation + +The corpus lives in its own store, `~/.local/share/knowledge-rag-archwiki`, not +the shared `~/.local/share/knowledge-rag`. Roughly 27k Arch chunks would +otherwise dominate both the BM25 index and the embedding space of any other +corpus kept there. `claude-rag` and `claude-archwiki` are separate sessions +against separate indexes. + +## Refresh guards + +The pacman hook fires unattended, so `reindex.py` refuses to run when either: + +- a `claude-archwiki` session already holds the store's instance lock + (`knowledge-rag`'s own lock is taken only by its `main()`, so importing the + orchestrator directly would bypass it), or +- the machine is already under load (1-minute load average above 1.5 per core). + +The load guard, not a GPU one: `knowledge-rag` ships the CPU-only +`onnxruntime` build, whose execution providers are `AzureExecutionProvider` and +`CPUExecutionProvider`. There is no CUDA provider, so despite the 3090 in this +box the embedding pass runs on the CPU and saturates roughly every core for +several minutes. A VRAM check would have looked reassuring and never fired. + +Deferring is safe: the converted Markdown is already on disk, so the next run +picks up exactly where this one stopped. diff --git a/python_pkg/archwiki_rag/__init__.py b/python_pkg/archwiki_rag/__init__.py new file mode 100644 index 00000000..f58c16d3 --- /dev/null +++ b/python_pkg/archwiki_rag/__init__.py @@ -0,0 +1,21 @@ +"""Offline Arch Wiki corpus for the ``knowledge-rag`` MCP server. + +The ``arch-wiki-docs`` package ships the whole ArchWiki as HTML under +``/usr/share/doc/arch-wiki/html/en``. ``knowledge-rag`` cannot index HTML -- +its parser dispatch table only understands ``.md``, ``.txt``, ``.pdf`` and a +handful of code suffixes -- so this package converts each page to Markdown and +drops it into a dedicated knowledge-rag store. + +Markdown specifically, not plain text: knowledge-rag chunks ``.md`` files with +a heading-aware splitter and everything else with a blind fixed-width one. +ArchWiki pages are heavily sectioned, so emitting real ``##``/``###`` headings +is what makes chunk boundaries land on section breaks instead of mid-sentence. + +The other load-bearing detail is in :mod:`python_pkg.archwiki_rag.sync`: the +knowledge-rag incremental reindex decides a file is unchanged by comparing +mtime and size, not content. The wiki dump regenerates every file on each +release, so the converter must leave byte-identical pages untouched on disk or +every ``arch-wiki-docs`` upgrade would re-embed the entire corpus. +""" + +from __future__ import annotations diff --git a/python_pkg/archwiki_rag/__main__.py b/python_pkg/archwiki_rag/__main__.py new file mode 100644 index 00000000..be3bb5ab --- /dev/null +++ b/python_pkg/archwiki_rag/__main__.py @@ -0,0 +1,14 @@ +"""Module entry point for the offline Arch Wiki RAG corpus builder. + +Usage: + PYTHONPATH=~/testsAndMisc python3 -m python_pkg.archwiki_rag --help +""" + +from __future__ import annotations + +import sys + +from python_pkg.archwiki_rag.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python_pkg/archwiki_rag/_reindex_entry.py b/python_pkg/archwiki_rag/_reindex_entry.py new file mode 100644 index 00000000..1796d0ff --- /dev/null +++ b/python_pkg/archwiki_rag/_reindex_entry.py @@ -0,0 +1,77 @@ +"""Headless reindex entry point, executed by the *knowledge-rag* interpreter. + +This module is deliberately tiny and never imported by the rest of the package: +``chromadb`` and the embedding model live in the knowledge-rag venv, not in the +testsAndMisc one, so :mod:`python_pkg.archwiki_rag.reindex` runs this file with +that other interpreter. + +It exists as a real file rather than a ``python -c`` string so the repo's ruff, +mypy and pytest still apply to it. +""" + +from __future__ import annotations + +import importlib +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + +_SERVER_MODULE = "mcp_server.server" + + +def _load_get_orchestrator() -> Callable[[], object]: + """Resolve ``mcp_server.server.get_orchestrator`` at call time. + + The import has to be deferred: ``mcp_server`` lives in the knowledge-rag + venv, so at module-import time under any other interpreter it is simply + absent. :func:`importlib.import_module` expresses that deliberately, + rather than a function-level ``import`` statement that reads like an + oversight and trips the lint that forbids one. + + Returns: + Callable[[], object]: The orchestrator factory. + + Raises: + ImportError: If knowledge-rag is not importable from this interpreter. + """ + return importlib.import_module(_SERVER_MODULE).get_orchestrator + + +def main() -> int: + """Run a synchronous smart reindex of the store named by ``KNOWLEDGE_RAG_DIR``. + + ``reindex_all`` is the same code path the ``reindex_documents(force=True)`` + MCP tool wraps, minus the background thread -- running it inline lets the + caller block until the embedding pass is genuinely finished. + + Returns: + int: 0 on success, 1 when knowledge-rag could not be imported or the + reindex raised. + """ + try: + get_orchestrator = _load_get_orchestrator() + except ImportError as exc: + sys.stderr.write(f"knowledge-rag is not importable: {exc}\n") + return 1 + + # This module is a process entry point: the caller sees an exit code and + # stderr, nothing else. Every failure the embedding backend can raise -- + # chromadb errors, model-download failures, OSError on a half-written + # index -- has to become exit 1 rather than a traceback. Re-raising as + # SystemExit keeps that translation explicit: the exception is reported + # and converted, never silently swallowed, which is what the lint against + # broad handlers is actually guarding against. + try: + stats = get_orchestrator().reindex_all() + except Exception as exc: + sys.stderr.write(f"reindex failed: {exc}\n") + raise SystemExit(1) from exc + + sys.stdout.write(f"reindex complete: {stats}\n") + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/python_pkg/archwiki_rag/cli.py b/python_pkg/archwiki_rag/cli.py new file mode 100644 index 00000000..34c9d767 --- /dev/null +++ b/python_pkg/archwiki_rag/cli.py @@ -0,0 +1,118 @@ +"""Command-line entry point for the offline Arch Wiki RAG corpus. + +Usage: + PYTHONPATH=~/testsAndMisc python -m python_pkg.archwiki_rag sync + PYTHONPATH=~/testsAndMisc python -m python_pkg.archwiki_rag sync --reindex +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +from python_pkg.archwiki_rag import reindex as reindex_mod +from python_pkg.archwiki_rag.constants import ( + DEFAULT_STORE_DIR, + DOCUMENTS_SUBDIR, + WIKI_HTML_DIR, +) +from python_pkg.archwiki_rag.sync import sync_pages + + +def _emit(message: str) -> None: + """Write a line to stdout. + + Parameters: + message (str): Line to write, without a trailing newline. + """ + sys.stdout.write(f"{message}\n") + + +def build_parser() -> argparse.ArgumentParser: + """Construct the argument parser. + + Returns: + argparse.ArgumentParser: Parser with the ``sync`` subcommand. + """ + parser = argparse.ArgumentParser( + prog="python -m python_pkg.archwiki_rag", + description="Convert the offline ArchWiki dump into a knowledge-rag corpus.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + sync_parser = subparsers.add_parser( + "sync", + help="convert the HTML dump to Markdown, optionally reindexing after", + ) + sync_parser.add_argument( + "--source", + type=Path, + default=WIKI_HTML_DIR, + help=f"offline HTML tree (default: {WIKI_HTML_DIR})", + ) + sync_parser.add_argument( + "--store", + type=Path, + default=DEFAULT_STORE_DIR, + help=f"knowledge-rag store to populate (default: {DEFAULT_STORE_DIR})", + ) + sync_parser.add_argument( + "--reindex", + action="store_true", + help="reindex afterwards when any page changed", + ) + sync_parser.add_argument( + "--ignore-load", + action="store_true", + help="reindex even when the machine is already under load", + ) + return parser + + +def run_sync(args: argparse.Namespace) -> int: + """Execute the ``sync`` subcommand. + + Parameters: + args (argparse.Namespace): Parsed arguments. + + Returns: + int: Process exit status. + """ + source: Path = args.source + if not source.is_dir(): + _emit(f"error: no HTML dump at {source} (is arch-wiki-docs installed?)") + return 1 + + result = sync_pages(source, args.store / DOCUMENTS_SUBDIR) + _emit( + f"converted {result.converted}, changed {result.changed}, " + f"skipped {result.skipped}", + ) + + if not args.reindex: + return 0 + if result.changed == 0: + _emit("nothing changed; skipping reindex") + return 0 + + reason = reindex_mod.blocking_reason(args.store, ignore_load=args.ignore_load) + if reason is not None: + _emit(f"deferring reindex: {reason}") + return 0 + + _emit("reindexing...") + return reindex_mod.run_reindex(args.store) + + +def main(argv: list[str] | None = None) -> int: + """Parse arguments and dispatch. + + Parameters: + argv (list[str] | None): Argument vector, defaulting to ``sys.argv[1:]``. + + Returns: + int: Process exit status. + """ + args = build_parser().parse_args(argv) + return run_sync(args) diff --git a/python_pkg/archwiki_rag/constants.py b/python_pkg/archwiki_rag/constants.py new file mode 100644 index 00000000..ee979b45 --- /dev/null +++ b/python_pkg/archwiki_rag/constants.py @@ -0,0 +1,68 @@ +"""Filesystem paths, selectors and thresholds for the Arch Wiki RAG corpus.""" + +from __future__ import annotations + +from pathlib import Path +import re + +# Source: the offline HTML dump shipped by the `arch-wiki-docs` package. +# Only the English tree is indexed; the other languages roughly quadruple the +# corpus without adding information this system would ever be queried for. +WIKI_HTML_DIR = Path("/usr/share/doc/arch-wiki/html/en") + +# Destination: a *dedicated* knowledge-rag store. Deliberately not the shared +# ~/.local/share/knowledge-rag one -- roughly 35k Arch chunks would dominate +# both the BM25 index and the embedding space of any other corpus kept there. +DEFAULT_STORE_DIR = Path.home() / ".local/share/knowledge-rag-archwiki" + +# Layout knowledge-rag imposes on a store directory. +DOCUMENTS_SUBDIR = "documents" +DATA_SUBDIR = "data" + +# Written by knowledge-rag's single-instance guard; first line is the pid. +LOCK_FILENAME = "knowledge-rag.lock" + +# The knowledge-rag venv, which owns chromadb/fastembed. The reindex must run +# under this interpreter, not the testsAndMisc one. +KNOWLEDGE_RAG_PYTHON = Path.home() / ".local/share/uv/tools/knowledge-rag/bin/python" + +LOADAVG_PATH = Path("/proc/loadavg") + +# Embedding is CPU-bound, not GPU-bound: knowledge-rag ships the CPU-only +# `onnxruntime` build, whose execution providers are Azure and CPU -- there is +# no CUDA provider, so a VRAM check would never fire. The reindex saturates +# roughly every core for minutes, so the guard watches load average instead. +# +# Expressed per core, so it is meaningful on any machine. Above this the box is +# already working hard (a build, a game, a download) and an unattended reindex +# should wait rather than pile on. +LOAD_PER_CORE_THRESHOLD = 1.5 + +# MediaWiki's Vector skin wraps the article body in this element. +CONTENT_ID = "mw-content-text" + +# Chrome that carries no information once the page is out of a browser. +STRIP_SELECTORS = ( + "#toc", + ".toc", + ".mw-editsection", + ".mw-jump-link", + ".navbox", + ".printfooter", + "#catlinks", + "script", + "style", +) + +WIKI_BASE_URL = "https://wiki.archlinux.org/title/" + +# A handful of files in en/ are named after an md5 digest rather than a page +# title; they are stray non-English pages (e.g. a Latvian "Main page") that the +# dump generator failed to file under its own language tree. +HASHED_STEM = re.compile(r"^[0-9a-f]{32}$") + +# Local hrefs look like "../en/Systemd/User.html#Basic_setup". Capture the page +# path so it can be rewritten to a citable upstream URL. +LOCAL_HREF = re.compile( + r"^(?:\.\./)*(?:[a-z]{2,3}(?:-[a-z]+)?/)?(?P.+?)\.html(?P#.*)?$", +) diff --git a/python_pkg/archwiki_rag/convert.py b/python_pkg/archwiki_rag/convert.py new file mode 100644 index 00000000..a335b37b --- /dev/null +++ b/python_pkg/archwiki_rag/convert.py @@ -0,0 +1,142 @@ +"""Convert an offline ArchWiki HTML page into RAG-friendly Markdown. + +Pure functions over strings: nothing here touches the filesystem, so the whole +conversion is testable from an HTML fixture. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from bs4 import BeautifulSoup, Tag +from markdownify import markdownify + +from python_pkg.archwiki_rag.constants import ( + CONTENT_ID, + HASHED_STEM, + LOCAL_HREF, + STRIP_SELECTORS, + WIKI_BASE_URL, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def is_wiki_page(path: Path) -> bool: + """Report whether a file in the dump is a real, indexable wiki page. + + Parameters: + path (Path): Candidate file from the offline HTML tree. + + Returns: + bool: True for ordinary pages, False for the md5-named stray non-English + pages the dump generator leaves in the English tree. + """ + return path.suffix == ".html" and not HASHED_STEM.match(path.stem) + + +def page_name(path: Path, source: Path) -> str: + """Derive the wiki page name from a file's position in the dump. + + Subpages are stored in nested directories -- ``en/Systemd/User.html`` is + the page ``Systemd/User`` -- so the name is the path relative to the tree + root, not merely the filename stem. + + Parameters: + path (Path): File such as ``.../en/Systemd/User.html``. + source (Path): Root of the offline HTML tree. + + Returns: + str: The page name, e.g. ``Systemd/User``. + """ + return path.relative_to(source).with_suffix("").as_posix() + + +def wiki_url(page: str) -> str: + """Build the upstream URL for a page name. + + Parameters: + page (str): Page name such as ``Systemd/User``. + + Returns: + str: Canonical https URL on wiki.archlinux.org. + """ + return f"{WIKI_BASE_URL}{page}" + + +def extract_title(soup: BeautifulSoup, fallback: str) -> str: + """Read the human-readable page title. + + Prefers the rendered ``

`` heading, which carries the wiki's own + capitalisation (``systemd``, not ``Systemd``). + + Parameters: + soup (BeautifulSoup): Parsed page. + fallback (str): Value to use when the page has no ``

``. + + Returns: + str: Page title. + """ + heading = soup.find("h1") + if heading is None: + return fallback + text = heading.get_text(strip=True) + return text or fallback + + +def rewrite_links(content: Tag) -> None: + """Rewrite relative page links to absolute wiki.archlinux.org URLs, in place. + + The dump cross-links pages as ``../en/Foo.html``, which is meaningless once + a chunk is quoted back to the user. Absolute URLs make retrieved passages + citable and let the reader follow references without the local tree. + + Parameters: + content (Tag): Article body to mutate. + """ + for anchor in content.find_all("a", href=True): + match = LOCAL_HREF.match(str(anchor["href"])) + if match is None: + continue + anchor["href"] = wiki_url(match["page"]) + (match["frag"] or "") + + +def strip_chrome(content: Tag) -> None: + """Remove navigation and editing furniture from the article body, in place. + + Parameters: + content (Tag): Article body to mutate. + """ + for selector in STRIP_SELECTORS: + for element in content.select(selector): + element.decompose() + + +def html_to_markdown(html: str, page: str) -> str | None: + """Render one wiki page as Markdown with a citable front-matter header. + + Parameters: + html (str): Raw HTML of an offline wiki page. + page (str): Page name used for the source URL and title fallback. + + Returns: + str | None: Markdown document, or None when the page has no article body + (malformed or placeholder files in the dump). + """ + soup = BeautifulSoup(html, "html.parser") + content = soup.find(id=CONTENT_ID) + if content is None: + return None + + title = extract_title(soup, page) + strip_chrome(content) + rewrite_links(content) + + body = markdownify(str(content), heading_style="ATX").strip() + + # A single H1 plus a source line: the title gives the heading-aware chunker + # a document-level anchor, and the URL rides along into every chunk's text + # so retrieved passages can be attributed without a metadata lookup. + header = f"# {title}\n\nSource: {wiki_url(page)}\n" + return f"{header}\n{body}\n" diff --git a/python_pkg/archwiki_rag/install.sh b/python_pkg/archwiki_rag/install.sh new file mode 100755 index 00000000..07168a39 --- /dev/null +++ b/python_pkg/archwiki_rag/install.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# ============================================================================ +# Install the offline Arch Wiki RAG corpus and its refresh hook. +# +# Installs dependencies, builds the corpus, and registers a pacman hook that +# refreshes it whenever `arch-wiki-docs` is upgraded. +# ============================================================================ + +set -euo pipefail + +SCRIPT_NAME="$(basename "$0")" +PKG_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${PKG_DIR}/../.." && pwd)" +readonly SCRIPT_NAME PKG_DIR REPO_ROOT +readonly STORE_DIR="${HOME}/.local/share/knowledge-rag-archwiki" +readonly REFRESH_BIN="/usr/local/bin/archwiki-rag-refresh" +readonly HOOK_PATH="/etc/pacman.d/hooks/95-archwiki-rag.hook" +readonly MCP_CONFIG="${HOME}/.claude/mcp-optional/archwiki.json" + +usage() { + echo "Usage: $SCRIPT_NAME [--no-hook]" + echo "Options:" + echo " --no-hook Install dependencies and build the corpus, but skip the" + echo " pacman hook (which needs sudo)." + echo " -h, --help Show this help" + exit 0 +} + +INSTALL_HOOK=1 + +install_dependencies() { + echo "==> Installing dependencies" + # arch-wiki-docs is the corpus itself; the two Python packages do the + # HTML -> Markdown conversion. + sudo pacman -S --needed --noconfirm \ + arch-wiki-docs \ + python-beautifulsoup4 \ + python-markdownify +} + +write_mcp_config() { + echo "==> Writing MCP config: $MCP_CONFIG" + mkdir -p "$(dirname "$MCP_CONFIG")" + cat >"$MCP_CONFIG" < Building corpus (this takes ~1 min for conversion)" + PYTHONPATH="$REPO_ROOT" python3 -m python_pkg.archwiki_rag sync --reindex +} + +install_hook() { + echo "==> Installing pacman hook" + + # The hook runs as root; this wrapper drops back to the invoking user and + # detaches the CPU-bound reindex so the pacman transaction is not blocked + # while the corpus re-embeds. + sudo tee "$REFRESH_BIN" >/dev/null </dev/null < Skipping pacman hook (--no-hook)" + fi + + echo "==> Done. Start a session with: claude-archwiki" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --no-hook) + INSTALL_HOOK=0 + shift + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +main "$@" diff --git a/python_pkg/archwiki_rag/reindex.py b/python_pkg/archwiki_rag/reindex.py new file mode 100644 index 00000000..dc528e0b --- /dev/null +++ b/python_pkg/archwiki_rag/reindex.py @@ -0,0 +1,131 @@ +"""Trigger a knowledge-rag reindex safely from outside the MCP server. + +Two guards wrap the reindex, both of which exist because this runs unattended +from a pacman hook: + +* **Instance lock.** knowledge-rag's single-instance lock is taken only by its + ``main()``. Importing the orchestrator directly bypasses it, so a reindex + fired while a ``claude-archwiki`` session is live would have two processes + writing one Chroma collection. The lock file is checked explicitly instead. +* **Machine load.** Embedding is CPU-bound -- knowledge-rag ships the CPU-only + ``onnxruntime`` build, so despite the 3090 in this box the pass saturates + roughly every core for minutes. If the machine is already working hard, + defer rather than compete: the converted Markdown stays on disk and the next + run picks up exactly where this one stopped. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess + +from python_pkg.archwiki_rag.constants import ( + DATA_SUBDIR, + KNOWLEDGE_RAG_PYTHON, + LOAD_PER_CORE_THRESHOLD, + LOADAVG_PATH, + LOCK_FILENAME, +) + +_ENTRY_MODULE = "python_pkg.archwiki_rag._reindex_entry" + + +def read_lock_pid(store_dir: Path) -> int | None: + """Read the pid recorded in the knowledge-rag instance lock. + + Parameters: + store_dir (Path): Root of the knowledge-rag store. + + Returns: + int | None: The pid, or None when there is no readable lock file. + """ + lock_path = store_dir / DATA_SUBDIR / LOCK_FILENAME + try: + first_line = lock_path.read_text(encoding="utf-8").strip().splitlines()[0] + return int(first_line) + except (IndexError, OSError, ValueError): + return None + + +def is_server_running(store_dir: Path) -> bool: + """Report whether a knowledge-rag MCP server currently owns this store. + + Parameters: + store_dir (Path): Root of the knowledge-rag store. + + Returns: + bool: True when the lock names a live process. A lock left behind by a + crashed server is treated as free. + """ + pid = read_lock_pid(store_dir) + return pid is not None and Path(f"/proc/{pid}").exists() + + +def load_per_core() -> float: + """Current 1-minute load average, divided by the CPU count. + + A single read of ``/proc/loadavg`` -- no forks, per the repo's polling + guidance. + + Returns: + float: Load per core, or 0.0 when ``/proc/loadavg`` is unreadable (a + machine that cannot report load is never treated as busy). + """ + try: + first_field = LOADAVG_PATH.read_text(encoding="utf-8").split(maxsplit=1)[0] + load = float(first_field) + except (IndexError, OSError, ValueError): + return 0.0 + return load / (os.cpu_count() or 1) + + +def blocking_reason(store_dir: Path, *, ignore_load: bool = False) -> str | None: + """Explain why a reindex should not run right now. + + Parameters: + store_dir (Path): Root of the knowledge-rag store. + ignore_load (bool): Skip the load check, for an explicitly requested run. + + Returns: + str | None: Human-readable reason to defer, or None when clear to proceed. + """ + if is_server_running(store_dir): + return "a knowledge-rag server is already using this store" + if not ignore_load: + load = load_per_core() + if load > LOAD_PER_CORE_THRESHOLD: + return ( + f"machine is busy (load {load:.2f}/core); deferring to avoid contention" + ) + return None + + +def run_reindex(store_dir: Path) -> int: + """Run the reindex under the knowledge-rag interpreter. + + Parameters: + store_dir (Path): Root of the knowledge-rag store, passed through as + ``KNOWLEDGE_RAG_DIR`` so the child targets the right corpus. + + Returns: + int: Exit status of the child process; 127 when its interpreter is absent. + """ + if not KNOWLEDGE_RAG_PYTHON.exists(): + return 127 + + env = dict(os.environ) + env["KNOWLEDGE_RAG_DIR"] = str(store_dir) + # The entry module imports from python_pkg, which is not installed into the + # knowledge-rag venv, so point that interpreter back at this repo. + repo_root = Path(__file__).resolve().parents[2] + env["PYTHONPATH"] = os.pathsep.join( + [str(repo_root), *([env["PYTHONPATH"]] if env.get("PYTHONPATH") else [])], + ) + + completed = subprocess.run( + [str(KNOWLEDGE_RAG_PYTHON), "-m", _ENTRY_MODULE], + env=env, + check=False, + ) + return completed.returncode diff --git a/python_pkg/archwiki_rag/sync.py b/python_pkg/archwiki_rag/sync.py new file mode 100644 index 00000000..2b553182 --- /dev/null +++ b/python_pkg/archwiki_rag/sync.py @@ -0,0 +1,109 @@ +"""Convert the offline ArchWiki dump into a knowledge-rag documents directory. + +The one rule that matters here: **never rewrite a file whose content did not +change.** knowledge-rag's incremental reindex decides a document is up to date +by comparing the stored mtime and size against the file on disk +(``server.py`` in the knowledge-rag package). The ``arch-wiki-docs`` dump +regenerates all ~2400 files on every release, so a converter that wrote +unconditionally would bump every mtime and force a full re-embed of the corpus +on each upgrade. Comparing rendered output against what is already on disk +turns that into a handful of pages. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from python_pkg.archwiki_rag.convert import html_to_markdown, is_wiki_page, page_name + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + +@dataclass(frozen=True) +class SyncResult: + """Outcome of one conversion pass over the dump. + + Attributes: + converted (int): Pages successfully rendered to Markdown. + changed (int): Pages whose Markdown differed from what was on disk and so + were written. Only these get re-embedded on the next reindex. + skipped (int): Files ignored -- md5-named strays and pages with no body. + """ + + converted: int + changed: int + skipped: int + + +def iter_pages(source: Path) -> Iterator[Path]: + """Yield indexable HTML pages from the dump, in a stable order. + + Walks recursively: subpages such as ``Systemd/User`` live in nested + directories, and they are some of the most useful pages in the wiki. + + Parameters: + source (Path): Directory holding the offline HTML tree. + + Yields: + Path: Each real wiki page, sorted so runs are reproducible. + """ + for path in sorted(source.rglob("*.html")): + if is_wiki_page(path): + yield path + + +def write_if_changed(destination: Path, text: str) -> bool: + """Write ``text`` only when it differs from the file already on disk. + + Parameters: + destination (Path): Target Markdown file. + text (str): Rendered document. + + Returns: + bool: True when the file was written, False when it was already identical. + """ + if destination.exists() and destination.read_text(encoding="utf-8") == text: + return False + destination.write_text(text, encoding="utf-8") + return True + + +def sync_pages(source: Path, documents_dir: Path) -> SyncResult: + """Convert every page in the dump into the knowledge-rag documents directory. + + Parameters: + source (Path): Offline HTML tree, e.g. ``/usr/share/doc/arch-wiki/html/en``. + documents_dir (Path): knowledge-rag ``documents/`` directory to populate. + + Returns: + SyncResult: Counts of converted, changed and skipped pages. + """ + documents_dir.mkdir(parents=True, exist_ok=True) + + converted = 0 + changed = 0 + skipped = 0 + + for path in sorted(source.rglob("*.html")): + if not is_wiki_page(path): + skipped += 1 + continue + page = page_name(path, source) + markdown = html_to_markdown( + path.read_text(encoding="utf-8", errors="replace"), page + ) + if markdown is None: + skipped += 1 + continue + converted += 1 + # Mirror the dump's directory layout; knowledge-rag walks documents/ + # recursively, so nesting costs nothing and keeps page names natural. + destination = documents_dir / f"{page}.md" + destination.parent.mkdir(parents=True, exist_ok=True) + if write_if_changed(destination, markdown): + changed += 1 + + return SyncResult(converted=converted, changed=changed, skipped=skipped) diff --git a/python_pkg/archwiki_rag/tests/__init__.py b/python_pkg/archwiki_rag/tests/__init__.py new file mode 100644 index 00000000..34cf7147 --- /dev/null +++ b/python_pkg/archwiki_rag/tests/__init__.py @@ -0,0 +1,3 @@ +"""Tests for the offline Arch Wiki RAG corpus builder.""" + +from __future__ import annotations diff --git a/python_pkg/archwiki_rag/tests/test_cli.py b/python_pkg/archwiki_rag/tests/test_cli.py new file mode 100644 index 00000000..573cd037 --- /dev/null +++ b/python_pkg/archwiki_rag/tests/test_cli.py @@ -0,0 +1,161 @@ +"""Tests for archwiki_rag.cli.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from python_pkg.archwiki_rag import cli +from python_pkg.archwiki_rag.sync import SyncResult + +MOD = "python_pkg.archwiki_rag.cli" + + +class TestBuildParser: + def test_requires_a_subcommand(self) -> None: + with pytest.raises(SystemExit): + cli.build_parser().parse_args([]) + + def test_sync_defaults(self) -> None: + args = cli.build_parser().parse_args(["sync"]) + assert args.command == "sync" + assert args.reindex is False + assert args.ignore_load is False + + def test_sync_accepts_overrides(self) -> None: + args = cli.build_parser().parse_args( + ["sync", "--source", "/s", "--store", "/d", "--reindex", "--ignore-load"], + ) + assert args.source == Path("/s") + assert args.store == Path("/d") + assert args.reindex is True + assert args.ignore_load is True + + +class TestRunSync: + def test_missing_source_reports_error( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + args = cli.build_parser().parse_args( + ["sync", "--source", str(tmp_path / "absent"), "--store", str(tmp_path)], + ) + assert cli.run_sync(args) == 1 + assert "arch-wiki-docs" in capsys.readouterr().out + + @patch(f"{MOD}.sync_pages", return_value=SyncResult(3, 2, 1)) + def test_reports_counts( + self, + mock_sync: MagicMock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + args = cli.build_parser().parse_args( + ["sync", "--source", str(tmp_path), "--store", str(tmp_path)], + ) + assert cli.run_sync(args) == 0 + assert "converted 3, changed 2, skipped 1" in capsys.readouterr().out + + @patch(f"{MOD}.reindex_mod.run_reindex") + @patch(f"{MOD}.sync_pages", return_value=SyncResult(3, 0, 0)) + def test_skips_reindex_when_nothing_changed( + self, + mock_sync: MagicMock, + mock_run: MagicMock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + args = cli.build_parser().parse_args( + ["sync", "--source", str(tmp_path), "--store", str(tmp_path), "--reindex"], + ) + assert cli.run_sync(args) == 0 + assert "nothing changed" in capsys.readouterr().out + mock_run.assert_not_called() + + @patch(f"{MOD}.reindex_mod.run_reindex") + @patch( + f"{MOD}.reindex_mod.blocking_reason", + return_value="machine is busy (load 4.00/core)", + ) + @patch(f"{MOD}.sync_pages", return_value=SyncResult(3, 3, 0)) + def test_defers_when_blocked( + self, + mock_sync: MagicMock, + mock_reason: MagicMock, + mock_run: MagicMock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + args = cli.build_parser().parse_args( + ["sync", "--source", str(tmp_path), "--store", str(tmp_path), "--reindex"], + ) + assert cli.run_sync(args) == 0 + assert "deferring reindex: machine is busy" in capsys.readouterr().out + mock_run.assert_not_called() + + @patch(f"{MOD}.reindex_mod.run_reindex", return_value=0) + @patch(f"{MOD}.reindex_mod.blocking_reason", return_value=None) + @patch(f"{MOD}.sync_pages", return_value=SyncResult(3, 3, 0)) + def test_runs_reindex_when_clear( + self, + mock_sync: MagicMock, + mock_reason: MagicMock, + mock_run: MagicMock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + args = cli.build_parser().parse_args( + ["sync", "--source", str(tmp_path), "--store", str(tmp_path), "--reindex"], + ) + assert cli.run_sync(args) == 0 + assert "reindexing..." in capsys.readouterr().out + mock_run.assert_called_once_with(Path(str(tmp_path))) + + @patch(f"{MOD}.reindex_mod.run_reindex", return_value=127) + @patch(f"{MOD}.reindex_mod.blocking_reason", return_value=None) + @patch(f"{MOD}.sync_pages", return_value=SyncResult(3, 3, 0)) + def test_propagates_reindex_failure( + self, + mock_sync: MagicMock, + mock_reason: MagicMock, + mock_run: MagicMock, + tmp_path: Path, + ) -> None: + args = cli.build_parser().parse_args( + ["sync", "--source", str(tmp_path), "--store", str(tmp_path), "--reindex"], + ) + assert cli.run_sync(args) == 127 + + @patch(f"{MOD}.reindex_mod.blocking_reason", return_value=None) + @patch(f"{MOD}.reindex_mod.run_reindex", return_value=0) + @patch(f"{MOD}.sync_pages", return_value=SyncResult(1, 1, 0)) + def test_ignore_load_is_forwarded( + self, + mock_sync: MagicMock, + mock_run: MagicMock, + mock_reason: MagicMock, + tmp_path: Path, + ) -> None: + args = cli.build_parser().parse_args( + [ + "sync", + "--source", + str(tmp_path), + "--store", + str(tmp_path), + "--reindex", + "--ignore-load", + ], + ) + cli.run_sync(args) + assert mock_reason.call_args.kwargs["ignore_load"] is True + + +class TestMain: + @patch(f"{MOD}.run_sync", return_value=0) + def test_dispatches_to_run_sync(self, mock_run_sync: MagicMock) -> None: + assert cli.main(["sync"]) == 0 + mock_run_sync.assert_called_once() diff --git a/python_pkg/archwiki_rag/tests/test_convert.py b/python_pkg/archwiki_rag/tests/test_convert.py new file mode 100644 index 00000000..a478d634 --- /dev/null +++ b/python_pkg/archwiki_rag/tests/test_convert.py @@ -0,0 +1,142 @@ +"""Tests for archwiki_rag.convert.""" + +from __future__ import annotations + +from pathlib import Path + +from bs4 import BeautifulSoup + +from python_pkg.archwiki_rag import convert + +PAGE_HTML = """ +systemd - ArchWiki +

systemd

+
+
Contents
+ [edit] + +

Basic usage

+

See systemd/User and + udev.

+

External project page.

+ 
systemctl status
+
+ + +""" + + +class TestIsWikiPage: + def test_ordinary_page(self) -> None: + assert convert.is_wiki_page(Path("/dump/en/Systemd.html")) + + def test_md5_named_stray_rejected(self) -> None: + stray = Path("/dump/en/01973e61502a6abccee11d2419aae5e4.html") + assert not convert.is_wiki_page(stray) + + def test_non_html_rejected(self) -> None: + assert not convert.is_wiki_page(Path("/dump/en/ArchWikiOffline.css")) + + +class TestPageName: + def test_top_level(self) -> None: + source = Path("/dump/en") + assert convert.page_name(source / "Systemd.html", source) == "Systemd" + + def test_subpage_keeps_hierarchy(self) -> None: + source = Path("/dump/en") + page = convert.page_name(source / "Systemd" / "User.html", source) + assert page == "Systemd/User" + + +class TestWikiUrl: + def test_builds_upstream_url(self) -> None: + assert convert.wiki_url("Systemd/User") == ( + "https://wiki.archlinux.org/title/Systemd/User" + ) + + +class TestExtractTitle: + def test_uses_h1(self) -> None: + soup = BeautifulSoup("

systemd

", "html.parser") + assert convert.extract_title(soup, "Systemd") == "systemd" + + def test_falls_back_without_h1(self) -> None: + soup = BeautifulSoup("

no heading

", "html.parser") + assert convert.extract_title(soup, "Systemd") == "Systemd" + + def test_falls_back_on_empty_h1(self) -> None: + soup = BeautifulSoup("

", "html.parser") + assert convert.extract_title(soup, "Systemd") == "Systemd" + + +def _rewritten_href(html: str) -> str: + """Rewrite links in a one-anchor fragment and return the resulting href. + + Parameters: + html (str): HTML fragment containing exactly one anchor. + + Returns: + str: The anchor's href after rewriting. + """ + soup = BeautifulSoup(html, "html.parser") + convert.rewrite_links(soup) + anchor = soup.a + assert anchor is not None + return str(anchor["href"]) + + +class TestRewriteLinks: + def test_local_link_becomes_absolute(self) -> None: + href = _rewritten_href('udev') + assert href == "https://wiki.archlinux.org/title/Udev" + + def test_fragment_is_preserved(self) -> None: + href = _rewritten_href('x') + assert href == "https://wiki.archlinux.org/title/Systemd/User#Basic_setup" + + def test_external_link_untouched(self) -> None: + href = _rewritten_href('p') + assert href == "https://systemd.io/" + + +class TestStripChrome: + def test_removes_navigation_furniture(self) -> None: + soup = BeautifulSoup( + '
C

body

', + "html.parser", + ) + convert.strip_chrome(soup) + assert "toc" not in str(soup) + assert "x()" not in str(soup) + assert "body" in str(soup) + + +class TestHtmlToMarkdown: + def test_returns_none_without_article_body(self) -> None: + assert convert.html_to_markdown("nope", "X") is None + + def test_renders_header_and_body(self) -> None: + out = convert.html_to_markdown(PAGE_HTML, "Systemd") + assert out is not None + assert out.startswith("# systemd\n") + assert "Source: https://wiki.archlinux.org/title/Systemd" in out + + def test_uses_atx_headings_for_chunker(self) -> None: + out = convert.html_to_markdown(PAGE_HTML, "Systemd") + assert out is not None + assert "## Basic usage" in out + + def test_strips_chrome_and_rewrites_links(self) -> None: + out = convert.html_to_markdown(PAGE_HTML, "Systemd") + assert out is not None + assert "Contents" not in out + assert "[edit]" not in out + assert "https://wiki.archlinux.org/title/Systemd/User#Basic_setup" in out + assert "../en/" not in out + + def test_ends_with_single_newline(self) -> None: + out = convert.html_to_markdown(PAGE_HTML, "Systemd") + assert out is not None + assert out.endswith("\n") + assert not out.endswith("\n\n") diff --git a/python_pkg/archwiki_rag/tests/test_main_entry.py b/python_pkg/archwiki_rag/tests/test_main_entry.py new file mode 100644 index 00000000..ec1c4e1b --- /dev/null +++ b/python_pkg/archwiki_rag/tests/test_main_entry.py @@ -0,0 +1,22 @@ +"""Tests for the archwiki_rag module entry point.""" + +from __future__ import annotations + +import runpy +import sys +from unittest.mock import MagicMock, patch + +import pytest + + +class TestModuleEntry: + @patch("python_pkg.archwiki_rag.cli.main", return_value=0) + def test_runs_cli_main(self, mock_main: MagicMock) -> None: + with ( + patch.object(sys, "argv", ["python_pkg.archwiki_rag", "sync"]), + pytest.raises(SystemExit) as excinfo, + ): + runpy.run_module("python_pkg.archwiki_rag", run_name="__main__") + + assert excinfo.value.code == 0 + mock_main.assert_called_once() diff --git a/python_pkg/archwiki_rag/tests/test_reindex.py b/python_pkg/archwiki_rag/tests/test_reindex.py new file mode 100644 index 00000000..6f5da913 --- /dev/null +++ b/python_pkg/archwiki_rag/tests/test_reindex.py @@ -0,0 +1,157 @@ +"""Tests for archwiki_rag.reindex.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +from unittest.mock import MagicMock, patch + +from python_pkg.archwiki_rag import reindex +from python_pkg.archwiki_rag.constants import DATA_SUBDIR, LOCK_FILENAME + +MOD = "python_pkg.archwiki_rag.reindex" + + +def _write_lock(store: Path, contents: str) -> None: + """Create a knowledge-rag instance lock file. + + Parameters: + store (Path): Store root. + contents (str): Raw lock file body. + """ + data = store / DATA_SUBDIR + data.mkdir(parents=True, exist_ok=True) + (data / LOCK_FILENAME).write_text(contents, encoding="utf-8") + + +class TestReadLockPid: + def test_missing_lock(self, tmp_path: Path) -> None: + assert reindex.read_lock_pid(tmp_path) is None + + def test_reads_pid(self, tmp_path: Path) -> None: + _write_lock(tmp_path, "4321\n") + assert reindex.read_lock_pid(tmp_path) == 4321 + + def test_non_numeric_lock(self, tmp_path: Path) -> None: + _write_lock(tmp_path, "not-a-pid\n") + assert reindex.read_lock_pid(tmp_path) is None + + def test_empty_lock(self, tmp_path: Path) -> None: + _write_lock(tmp_path, "") + assert reindex.read_lock_pid(tmp_path) is None + + +class TestIsServerRunning: + def test_no_lock(self, tmp_path: Path) -> None: + assert reindex.is_server_running(tmp_path) is False + + def test_live_process(self, tmp_path: Path) -> None: + _write_lock(tmp_path, "1\n") # pid 1 always exists + assert reindex.is_server_running(tmp_path) is True + + def test_stale_lock_treated_as_free(self, tmp_path: Path) -> None: + _write_lock(tmp_path, "999999999\n") + assert reindex.is_server_running(tmp_path) is False + + +class TestLoadPerCore: + def test_divides_load_by_cpu_count(self, tmp_path: Path) -> None: + loadavg = tmp_path / "loadavg" + loadavg.write_text("8.00 4.00 2.00 3/1234 5678\n", encoding="utf-8") + with ( + patch(f"{MOD}.LOADAVG_PATH", loadavg), + patch(f"{MOD}.os.cpu_count", return_value=4), + ): + assert reindex.load_per_core() == 2.0 + + def test_missing_file_is_not_busy(self, tmp_path: Path) -> None: + with patch(f"{MOD}.LOADAVG_PATH", tmp_path / "absent"): + assert reindex.load_per_core() == 0.0 + + def test_unparseable_content_is_not_busy(self, tmp_path: Path) -> None: + loadavg = tmp_path / "loadavg" + loadavg.write_text("garbage\n", encoding="utf-8") + with patch(f"{MOD}.LOADAVG_PATH", loadavg): + assert reindex.load_per_core() == 0.0 + + def test_empty_file_is_not_busy(self, tmp_path: Path) -> None: + loadavg = tmp_path / "loadavg" + loadavg.write_text("", encoding="utf-8") + with patch(f"{MOD}.LOADAVG_PATH", loadavg): + assert reindex.load_per_core() == 0.0 + + def test_unknown_cpu_count_falls_back_to_one(self, tmp_path: Path) -> None: + loadavg = tmp_path / "loadavg" + loadavg.write_text("3.00 1.00 1.00 1/1 1\n", encoding="utf-8") + with ( + patch(f"{MOD}.LOADAVG_PATH", loadavg), + patch(f"{MOD}.os.cpu_count", return_value=None), + ): + assert reindex.load_per_core() == 3.0 + + +class TestBlockingReason: + @patch(f"{MOD}.is_server_running", return_value=True) + def test_server_running_blocks(self, mock_running: MagicMock) -> None: + reason = reindex.blocking_reason(Path("/store")) + assert reason is not None + assert "already using this store" in reason + + @patch(f"{MOD}.load_per_core", return_value=4.0) + @patch(f"{MOD}.is_server_running", return_value=False) + def test_busy_machine_blocks( + self, + mock_running: MagicMock, + mock_load: MagicMock, + ) -> None: + reason = reindex.blocking_reason(Path("/store")) + assert reason is not None + assert "machine is busy" in reason + + @patch(f"{MOD}.load_per_core", return_value=4.0) + @patch(f"{MOD}.is_server_running", return_value=False) + def test_ignore_load_overrides( + self, + mock_running: MagicMock, + mock_load: MagicMock, + ) -> None: + assert reindex.blocking_reason(Path("/store"), ignore_load=True) is None + + @patch(f"{MOD}.load_per_core", return_value=0.2) + @patch(f"{MOD}.is_server_running", return_value=False) + def test_idle_machine_is_clear( + self, + mock_running: MagicMock, + mock_load: MagicMock, + ) -> None: + assert reindex.blocking_reason(Path("/store")) is None + + +class TestRunReindex: + def test_missing_interpreter(self) -> None: + with patch(f"{MOD}.KNOWLEDGE_RAG_PYTHON", Path("/nonexistent/python")): + assert reindex.run_reindex(Path("/store")) == 127 + + @patch(f"{MOD}.subprocess.run") + def test_passes_store_and_repo_root(self, mock_run: MagicMock) -> None: + mock_run.return_value = subprocess.CompletedProcess([], returncode=0) + with ( + patch.object(Path, "exists", return_value=True), + patch.dict(f"{MOD}.os.environ", {}, clear=True), + ): + assert reindex.run_reindex(Path("/store")) == 0 + + env = mock_run.call_args.kwargs["env"] + assert env["KNOWLEDGE_RAG_DIR"] == "/store" + assert env["PYTHONPATH"].endswith("testsAndMisc") + + @patch(f"{MOD}.subprocess.run") + def test_prepends_to_existing_pythonpath(self, mock_run: MagicMock) -> None: + mock_run.return_value = subprocess.CompletedProcess([], returncode=3) + with ( + patch.object(Path, "exists", return_value=True), + patch.dict(f"{MOD}.os.environ", {"PYTHONPATH": "/pre"}, clear=True), + ): + assert reindex.run_reindex(Path("/store")) == 3 + + assert mock_run.call_args.kwargs["env"]["PYTHONPATH"].endswith(":/pre") diff --git a/python_pkg/archwiki_rag/tests/test_reindex_entry.py b/python_pkg/archwiki_rag/tests/test_reindex_entry.py new file mode 100644 index 00000000..f61b5d9d --- /dev/null +++ b/python_pkg/archwiki_rag/tests/test_reindex_entry.py @@ -0,0 +1,84 @@ +"""Tests for archwiki_rag._reindex_entry. + +The entry point normally runs under the knowledge-rag interpreter, where +``mcp_server`` is importable. Here a stub module is injected into +``sys.modules`` so the three exit paths can be exercised in-process. +""" + +from __future__ import annotations + +import sys +import types +from typing import TYPE_CHECKING, cast +from unittest.mock import MagicMock + +import pytest + +from python_pkg.archwiki_rag import _reindex_entry + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@pytest.fixture +def fake_mcp_server() -> Iterator[MagicMock]: + """Install a stub ``mcp_server.server`` module for the duration of a test. + + Yields: + MagicMock: The stubbed ``get_orchestrator`` callable. + """ + package = types.ModuleType("mcp_server") + server = types.ModuleType("mcp_server.server") + get_orchestrator = MagicMock() + server.get_orchestrator = get_orchestrator + package.server = server + + sys.modules["mcp_server"] = package + sys.modules["mcp_server.server"] = server + try: + yield get_orchestrator + finally: + del sys.modules["mcp_server.server"] + del sys.modules["mcp_server"] + + +class TestMain: + def test_reports_missing_knowledge_rag( + self, + capsys: pytest.CaptureFixture[str], + ) -> None: + # A None entry in sys.modules makes `import` raise ImportError, which + # is exactly what happens for real when this module runs outside the + # knowledge-rag venv. + sys.modules["mcp_server"] = cast("types.ModuleType", None) + try: + assert _reindex_entry.main() == 1 + finally: + del sys.modules["mcp_server"] + + assert "not importable" in capsys.readouterr().err + + def test_reports_reindex_failure( + self, + fake_mcp_server: MagicMock, + capsys: pytest.CaptureFixture[str], + ) -> None: + fake_mcp_server.return_value.reindex_all.side_effect = RuntimeError("chroma") + + # A backend failure is translated into a non-zero exit rather than a + # traceback, so the subprocess caller sees a clean exit code. + with pytest.raises(SystemExit) as excinfo: + _reindex_entry.main() + + assert excinfo.value.code == 1 + assert "reindex failed: chroma" in capsys.readouterr().err + + def test_success_reports_stats( + self, + fake_mcp_server: MagicMock, + capsys: pytest.CaptureFixture[str], + ) -> None: + fake_mcp_server.return_value.reindex_all.return_value = {"indexed": 7} + + assert _reindex_entry.main() == 0 + assert "reindex complete" in capsys.readouterr().out diff --git a/python_pkg/archwiki_rag/tests/test_sync.py b/python_pkg/archwiki_rag/tests/test_sync.py new file mode 100644 index 00000000..b4741d77 --- /dev/null +++ b/python_pkg/archwiki_rag/tests/test_sync.py @@ -0,0 +1,125 @@ +"""Tests for archwiki_rag.sync.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.archwiki_rag import sync + +if TYPE_CHECKING: + from pathlib import Path + +PAGE = """ +

{title}

+

Body

{body}

+ +""" + +NO_BODY = "

placeholder

" + + +def _write_page(path: Path, title: str, body: str = "text") -> None: + """Write a minimal wiki page fixture. + + Parameters: + path (Path): Destination HTML file. + title (str): Page heading. + body (str): Paragraph content. + """ + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(PAGE.format(title=title, body=body), encoding="utf-8") + + +class TestIterPages: + def test_skips_strays_and_finds_subpages(self, tmp_path: Path) -> None: + _write_page(tmp_path / "Systemd.html", "systemd") + _write_page(tmp_path / "Systemd" / "User.html", "systemd/User") + (tmp_path / f"{'a' * 32}.html").write_text("stray", encoding="utf-8") + + found = sorted(p.name for p in sync.iter_pages(tmp_path)) + assert found == ["Systemd.html", "User.html"] + + +class TestWriteIfChanged: + def test_writes_new_file(self, tmp_path: Path) -> None: + target = tmp_path / "a.md" + assert sync.write_if_changed(target, "hello") is True + assert target.read_text(encoding="utf-8") == "hello" + + def test_rewrites_when_content_differs(self, tmp_path: Path) -> None: + target = tmp_path / "a.md" + target.write_text("old", encoding="utf-8") + assert sync.write_if_changed(target, "new") is True + assert target.read_text(encoding="utf-8") == "new" + + def test_leaves_identical_file_untouched(self, tmp_path: Path) -> None: + target = tmp_path / "a.md" + target.write_text("same", encoding="utf-8") + before = target.stat().st_mtime_ns + + assert sync.write_if_changed(target, "same") is False + # The whole incremental-reindex design rests on this: knowledge-rag + # compares mtime and size, so an unchanged page must not be touched. + assert target.stat().st_mtime_ns == before + + +class TestSyncPages: + def test_counts_converted_changed_and_skipped(self, tmp_path: Path) -> None: + source = tmp_path / "src" + _write_page(source / "Systemd.html", "systemd") + _write_page(source / "Systemd" / "User.html", "systemd/User") + (source / f"{'b' * 32}.html").write_text("stray", encoding="utf-8") + (source / "Broken.html").write_text(NO_BODY, encoding="utf-8") + + result = sync.sync_pages(source, tmp_path / "documents") + + assert result.converted == 2 + assert result.changed == 2 + assert result.skipped == 2 + + def test_preserves_subpage_hierarchy(self, tmp_path: Path) -> None: + source = tmp_path / "src" + _write_page(source / "Systemd" / "User.html", "systemd/User") + documents = tmp_path / "documents" + + sync.sync_pages(source, documents) + + page = documents / "Systemd" / "User.md" + assert page.exists() + assert "https://wiki.archlinux.org/title/Systemd/User" in page.read_text( + encoding="utf-8", + ) + + def test_second_run_changes_nothing(self, tmp_path: Path) -> None: + source = tmp_path / "src" + _write_page(source / "Systemd.html", "systemd") + documents = tmp_path / "documents" + + sync.sync_pages(source, documents) + mtimes = {p: p.stat().st_mtime_ns for p in documents.rglob("*.md")} + second = sync.sync_pages(source, documents) + + assert second.changed == 0 + assert all(p.stat().st_mtime_ns == m for p, m in mtimes.items()) + + def test_edited_page_is_rewritten(self, tmp_path: Path) -> None: + source = tmp_path / "src" + _write_page(source / "Systemd.html", "systemd", body="first") + documents = tmp_path / "documents" + sync.sync_pages(source, documents) + + _write_page(source / "Systemd.html", "systemd", body="second") + result = sync.sync_pages(source, documents) + + assert result.changed == 1 + assert "second" in (documents / "Systemd.md").read_text(encoding="utf-8") + + def test_creates_documents_directory(self, tmp_path: Path) -> None: + source = tmp_path / "src" + source.mkdir() + documents = tmp_path / "nested" / "documents" + + result = sync.sync_pages(source, documents) + + assert documents.is_dir() + assert result == sync.SyncResult(converted=0, changed=0, skipped=0)