mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 19:47:12 +01:00
Krzysztof kuhy Rudnicki
86565bdef6
Indexes the arch-wiki-docs dump that is already on disk and exposes it
to the knowledge-rag MCP server, so Arch questions can be asked by
symptom ("why doesn't my systemd user unit see DISPLAY under i3")
rather than by page title, with no network round-trip at query time.
The one Arch-specific MCP server in the wild fetches pages over the
network and bundles pacman operations that duplicate the local yay-mcp.
This does neither: it only searches, against a local corpus.
install.sh installs its own dependencies, wires the MCP config and
registers a pacman hook so the index follows package updates. The
generated index is machine-specific and regenerable, so it is
gitignored rather than committed.
_reindex_entry runs under the knowledge-rag interpreter, where
mcp_server is importable and this repo's packages are not. Resolving
that import with importlib.import_module states the deferral outright
instead of a function-level import statement that reads as an oversight,
and the backend failure path now re-raises as SystemExit, so the
exit-code translation is explicit rather than a swallowed exception.
Both replace lint suppressions; the package carries none.
pylint runs in its own venv, so beautifulsoup4 and markdownify are
declared there too — being in meta/requirements.txt does not make them
resolvable to the hook.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YQLwdquDCNmWtvgi3Lk72J
69 lines
2.6 KiB
Python
69 lines
2.6 KiB
Python
"""Filesystem paths, selectors and thresholds for the Arch Wiki RAG corpus."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
import re
|
|
|
|
# Source: the offline HTML dump shipped by the `arch-wiki-docs` package.
|
|
# Only the English tree is indexed; the other languages roughly quadruple the
|
|
# corpus without adding information this system would ever be queried for.
|
|
WIKI_HTML_DIR = Path("/usr/share/doc/arch-wiki/html/en")
|
|
|
|
# Destination: a *dedicated* knowledge-rag store. Deliberately not the shared
|
|
# ~/.local/share/knowledge-rag one -- roughly 35k Arch chunks would dominate
|
|
# both the BM25 index and the embedding space of any other corpus kept there.
|
|
DEFAULT_STORE_DIR = Path.home() / ".local/share/knowledge-rag-archwiki"
|
|
|
|
# Layout knowledge-rag imposes on a store directory.
|
|
DOCUMENTS_SUBDIR = "documents"
|
|
DATA_SUBDIR = "data"
|
|
|
|
# Written by knowledge-rag's single-instance guard; first line is the pid.
|
|
LOCK_FILENAME = "knowledge-rag.lock"
|
|
|
|
# The knowledge-rag venv, which owns chromadb/fastembed. The reindex must run
|
|
# under this interpreter, not the testsAndMisc one.
|
|
KNOWLEDGE_RAG_PYTHON = Path.home() / ".local/share/uv/tools/knowledge-rag/bin/python"
|
|
|
|
LOADAVG_PATH = Path("/proc/loadavg")
|
|
|
|
# Embedding is CPU-bound, not GPU-bound: knowledge-rag ships the CPU-only
|
|
# `onnxruntime` build, whose execution providers are Azure and CPU -- there is
|
|
# no CUDA provider, so a VRAM check would never fire. The reindex saturates
|
|
# roughly every core for minutes, so the guard watches load average instead.
|
|
#
|
|
# Expressed per core, so it is meaningful on any machine. Above this the box is
|
|
# already working hard (a build, a game, a download) and an unattended reindex
|
|
# should wait rather than pile on.
|
|
LOAD_PER_CORE_THRESHOLD = 1.5
|
|
|
|
# MediaWiki's Vector skin wraps the article body in this element.
|
|
CONTENT_ID = "mw-content-text"
|
|
|
|
# Chrome that carries no information once the page is out of a browser.
|
|
STRIP_SELECTORS = (
|
|
"#toc",
|
|
".toc",
|
|
".mw-editsection",
|
|
".mw-jump-link",
|
|
".navbox",
|
|
".printfooter",
|
|
"#catlinks",
|
|
"script",
|
|
"style",
|
|
)
|
|
|
|
WIKI_BASE_URL = "https://wiki.archlinux.org/title/"
|
|
|
|
# A handful of files in en/ are named after an md5 digest rather than a page
|
|
# title; they are stray non-English pages (e.g. a Latvian "Main page") that the
|
|
# dump generator failed to file under its own language tree.
|
|
HASHED_STEM = re.compile(r"^[0-9a-f]{32}$")
|
|
|
|
# Local hrefs look like "../en/Systemd/User.html#Basic_setup". Capture the page
|
|
# path so it can be rewritten to a citable upstream URL.
|
|
LOCAL_HREF = re.compile(
|
|
r"^(?:\.\./)*(?:[a-z]{2,3}(?:-[a-z]+)?/)?(?P<page>.+?)\.html(?P<frag>#.*)?$",
|
|
)
|