mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-11 00:27:01 +01:00
Krzysztof kuhy Rudnicki
86565bdef6
Indexes the arch-wiki-docs dump that is already on disk and exposes it
to the knowledge-rag MCP server, so Arch questions can be asked by
symptom ("why doesn't my systemd user unit see DISPLAY under i3")
rather than by page title, with no network round-trip at query time.
The one Arch-specific MCP server in the wild fetches pages over the
network and bundles pacman operations that duplicate the local yay-mcp.
This does neither: it only searches, against a local corpus.
install.sh installs its own dependencies, wires the MCP config and
registers a pacman hook so the index follows package updates. The
generated index is machine-specific and regenerable, so it is
gitignored rather than committed.
_reindex_entry runs under the knowledge-rag interpreter, where
mcp_server is importable and this repo's packages are not. Resolving
that import with importlib.import_module states the deferral outright
instead of a function-level import statement that reads as an oversight,
and the backend failure path now re-raises as SystemExit, so the
exit-code translation is explicit rather than a swallowed exception.
Both replace lint suppressions; the package carries none.
pylint runs in its own venv, so beautifulsoup4 and markdownify are
declared there too — being in meta/requirements.txt does not make them
resolvable to the hook.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YQLwdquDCNmWtvgi3Lk72J
110 lines
3.6 KiB
Python
110 lines
3.6 KiB
Python
"""Convert the offline ArchWiki dump into a knowledge-rag documents directory.
|
|
|
|
The one rule that matters here: **never rewrite a file whose content did not
|
|
change.** knowledge-rag's incremental reindex decides a document is up to date
|
|
by comparing the stored mtime and size against the file on disk
|
|
(``server.py`` in the knowledge-rag package). The ``arch-wiki-docs`` dump
|
|
regenerates all ~2400 files on every release, so a converter that wrote
|
|
unconditionally would bump every mtime and force a full re-embed of the corpus
|
|
on each upgrade. Comparing rendered output against what is already on disk
|
|
turns that into a handful of pages.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING
|
|
|
|
from python_pkg.archwiki_rag.convert import html_to_markdown, is_wiki_page, page_name
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SyncResult:
|
|
"""Outcome of one conversion pass over the dump.
|
|
|
|
Attributes:
|
|
converted (int): Pages successfully rendered to Markdown.
|
|
changed (int): Pages whose Markdown differed from what was on disk and so
|
|
were written. Only these get re-embedded on the next reindex.
|
|
skipped (int): Files ignored -- md5-named strays and pages with no body.
|
|
"""
|
|
|
|
converted: int
|
|
changed: int
|
|
skipped: int
|
|
|
|
|
|
def iter_pages(source: Path) -> Iterator[Path]:
|
|
"""Yield indexable HTML pages from the dump, in a stable order.
|
|
|
|
Walks recursively: subpages such as ``Systemd/User`` live in nested
|
|
directories, and they are some of the most useful pages in the wiki.
|
|
|
|
Parameters:
|
|
source (Path): Directory holding the offline HTML tree.
|
|
|
|
Yields:
|
|
Path: Each real wiki page, sorted so runs are reproducible.
|
|
"""
|
|
for path in sorted(source.rglob("*.html")):
|
|
if is_wiki_page(path):
|
|
yield path
|
|
|
|
|
|
def write_if_changed(destination: Path, text: str) -> bool:
|
|
"""Write ``text`` only when it differs from the file already on disk.
|
|
|
|
Parameters:
|
|
destination (Path): Target Markdown file.
|
|
text (str): Rendered document.
|
|
|
|
Returns:
|
|
bool: True when the file was written, False when it was already identical.
|
|
"""
|
|
if destination.exists() and destination.read_text(encoding="utf-8") == text:
|
|
return False
|
|
destination.write_text(text, encoding="utf-8")
|
|
return True
|
|
|
|
|
|
def sync_pages(source: Path, documents_dir: Path) -> SyncResult:
|
|
"""Convert every page in the dump into the knowledge-rag documents directory.
|
|
|
|
Parameters:
|
|
source (Path): Offline HTML tree, e.g. ``/usr/share/doc/arch-wiki/html/en``.
|
|
documents_dir (Path): knowledge-rag ``documents/`` directory to populate.
|
|
|
|
Returns:
|
|
SyncResult: Counts of converted, changed and skipped pages.
|
|
"""
|
|
documents_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
converted = 0
|
|
changed = 0
|
|
skipped = 0
|
|
|
|
for path in sorted(source.rglob("*.html")):
|
|
if not is_wiki_page(path):
|
|
skipped += 1
|
|
continue
|
|
page = page_name(path, source)
|
|
markdown = html_to_markdown(
|
|
path.read_text(encoding="utf-8", errors="replace"), page
|
|
)
|
|
if markdown is None:
|
|
skipped += 1
|
|
continue
|
|
converted += 1
|
|
# Mirror the dump's directory layout; knowledge-rag walks documents/
|
|
# recursively, so nesting costs nothing and keeps page names natural.
|
|
destination = documents_dir / f"{page}.md"
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
if write_if_changed(destination, markdown):
|
|
changed += 1
|
|
|
|
return SyncResult(converted=converted, changed=changed, skipped=skipped)
|