mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-11 00:27:01 +01:00
Krzysztof kuhy Rudnicki
86565bdef6
Indexes the arch-wiki-docs dump that is already on disk and exposes it
to the knowledge-rag MCP server, so Arch questions can be asked by
symptom ("why doesn't my systemd user unit see DISPLAY under i3")
rather than by page title, with no network round-trip at query time.
The one Arch-specific MCP server in the wild fetches pages over the
network and bundles pacman operations that duplicate the local yay-mcp.
This does neither: it only searches, against a local corpus.
install.sh installs its own dependencies, wires the MCP config and
registers a pacman hook so the index follows package updates. The
generated index is machine-specific and regenerable, so it is
gitignored rather than committed.
_reindex_entry runs under the knowledge-rag interpreter, where
mcp_server is importable and this repo's packages are not. Resolving
that import with importlib.import_module states the deferral outright
instead of a function-level import statement that reads as an oversight,
and the backend failure path now re-raises as SystemExit, so the
exit-code translation is explicit rather than a swallowed exception.
Both replace lint suppressions; the package carries none.
pylint runs in its own venv, so beautifulsoup4 and markdownify are
declared there too — being in meta/requirements.txt does not make them
resolvable to the hook.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YQLwdquDCNmWtvgi3Lk72J
119 lines
3.1 KiB
Python
119 lines
3.1 KiB
Python
"""Command-line entry point for the offline Arch Wiki RAG corpus.
|
|
|
|
Usage:
|
|
PYTHONPATH=~/testsAndMisc python -m python_pkg.archwiki_rag sync
|
|
PYTHONPATH=~/testsAndMisc python -m python_pkg.archwiki_rag sync --reindex
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from python_pkg.archwiki_rag import reindex as reindex_mod
|
|
from python_pkg.archwiki_rag.constants import (
|
|
DEFAULT_STORE_DIR,
|
|
DOCUMENTS_SUBDIR,
|
|
WIKI_HTML_DIR,
|
|
)
|
|
from python_pkg.archwiki_rag.sync import sync_pages
|
|
|
|
|
|
def _emit(message: str) -> None:
|
|
"""Write a line to stdout.
|
|
|
|
Parameters:
|
|
message (str): Line to write, without a trailing newline.
|
|
"""
|
|
sys.stdout.write(f"{message}\n")
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
"""Construct the argument parser.
|
|
|
|
Returns:
|
|
argparse.ArgumentParser: Parser with the ``sync`` subcommand.
|
|
"""
|
|
parser = argparse.ArgumentParser(
|
|
prog="python -m python_pkg.archwiki_rag",
|
|
description="Convert the offline ArchWiki dump into a knowledge-rag corpus.",
|
|
)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
sync_parser = subparsers.add_parser(
|
|
"sync",
|
|
help="convert the HTML dump to Markdown, optionally reindexing after",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--source",
|
|
type=Path,
|
|
default=WIKI_HTML_DIR,
|
|
help=f"offline HTML tree (default: {WIKI_HTML_DIR})",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--store",
|
|
type=Path,
|
|
default=DEFAULT_STORE_DIR,
|
|
help=f"knowledge-rag store to populate (default: {DEFAULT_STORE_DIR})",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--reindex",
|
|
action="store_true",
|
|
help="reindex afterwards when any page changed",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--ignore-load",
|
|
action="store_true",
|
|
help="reindex even when the machine is already under load",
|
|
)
|
|
return parser
|
|
|
|
|
|
def run_sync(args: argparse.Namespace) -> int:
|
|
"""Execute the ``sync`` subcommand.
|
|
|
|
Parameters:
|
|
args (argparse.Namespace): Parsed arguments.
|
|
|
|
Returns:
|
|
int: Process exit status.
|
|
"""
|
|
source: Path = args.source
|
|
if not source.is_dir():
|
|
_emit(f"error: no HTML dump at {source} (is arch-wiki-docs installed?)")
|
|
return 1
|
|
|
|
result = sync_pages(source, args.store / DOCUMENTS_SUBDIR)
|
|
_emit(
|
|
f"converted {result.converted}, changed {result.changed}, "
|
|
f"skipped {result.skipped}",
|
|
)
|
|
|
|
if not args.reindex:
|
|
return 0
|
|
if result.changed == 0:
|
|
_emit("nothing changed; skipping reindex")
|
|
return 0
|
|
|
|
reason = reindex_mod.blocking_reason(args.store, ignore_load=args.ignore_load)
|
|
if reason is not None:
|
|
_emit(f"deferring reindex: {reason}")
|
|
return 0
|
|
|
|
_emit("reindexing...")
|
|
return reindex_mod.run_reindex(args.store)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
"""Parse arguments and dispatch.
|
|
|
|
Parameters:
|
|
argv (list[str] | None): Argument vector, defaulting to ``sys.argv[1:]``.
|
|
|
|
Returns:
|
|
int: Process exit status.
|
|
"""
|
|
args = build_parser().parse_args(argv)
|
|
return run_sync(args)
|