mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-21 15:02:19 +01:00
Some checks are pending
md-naming / markdown naming convention (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (mypy, mypy) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (pylint, pylint) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (rest, ) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (shellcheck, shellcheck) (push) Waiting to run
Python tests / test (push) Waiting to run
repo-contract / repo contract (push) Waiting to run
Shell tests / shell-tests (push) Waiting to run
Shell tests / lib-tests (push) Waiting to run
Shell tests / arch-tests (push) Waiting to run
Twelve interactive CLIs end their --help on credit_line() from kuhy-credit v0.3.1 (android_ui, workout_firebase_flow, app_icons, archwiki_rag, artgate, code_tutor, js_anki, morning_routine, phone_lease, random_jpg, sql_drills, vscode_optimizer). The repo has no GUI, web app or game, so --help is the only surface. morning_routine's unit runs /usr/bin/python, so its install.sh now installs the requirements.txt pin there. Stdio/daemon entry points stay without it; the evidence file lists each with the reason. Also bumps Flutter to 3.50.1 and the stale Python pins. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
122 lines
3.2 KiB
Python
122 lines
3.2 KiB
Python
"""Command-line entry point for the offline Arch Wiki RAG corpus.
|
|
|
|
Usage:
|
|
PYTHONPATH=~/src/testsAndMisc python -m python_pkg.archwiki_rag sync
|
|
PYTHONPATH=~/src/testsAndMisc python -m python_pkg.archwiki_rag sync --reindex
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from kuhy_credit import credit_line
|
|
|
|
from python_pkg.archwiki_rag import reindex as reindex_mod
|
|
from python_pkg.archwiki_rag.constants import (
|
|
DEFAULT_STORE_DIR,
|
|
DOCUMENTS_SUBDIR,
|
|
WIKI_HTML_DIR,
|
|
)
|
|
from python_pkg.archwiki_rag.sync import sync_pages
|
|
|
|
|
|
def _emit(message: str) -> None:
|
|
"""Write a line to stdout.
|
|
|
|
Parameters:
|
|
message (str): Line to write, without a trailing newline.
|
|
"""
|
|
sys.stdout.write(f"{message}\n")
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
"""Construct the argument parser.
|
|
|
|
Returns:
|
|
argparse.ArgumentParser: Parser with the ``sync`` subcommand.
|
|
"""
|
|
parser = argparse.ArgumentParser(
|
|
prog="python -m python_pkg.archwiki_rag",
|
|
description="Convert the offline ArchWiki dump into a knowledge-rag corpus.",
|
|
epilog=credit_line(),
|
|
)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
sync_parser = subparsers.add_parser(
|
|
"sync",
|
|
help="convert the HTML dump to Markdown, optionally reindexing after",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--source",
|
|
type=Path,
|
|
default=WIKI_HTML_DIR,
|
|
help=f"offline HTML tree (default: {WIKI_HTML_DIR})",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--store",
|
|
type=Path,
|
|
default=DEFAULT_STORE_DIR,
|
|
help=f"knowledge-rag store to populate (default: {DEFAULT_STORE_DIR})",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--reindex",
|
|
action="store_true",
|
|
help="reindex afterwards when any page changed",
|
|
)
|
|
sync_parser.add_argument(
|
|
"--ignore-load",
|
|
action="store_true",
|
|
help="reindex even when the machine is already under load",
|
|
)
|
|
return parser
|
|
|
|
|
|
def run_sync(args: argparse.Namespace) -> int:
|
|
"""Execute the ``sync`` subcommand.
|
|
|
|
Parameters:
|
|
args (argparse.Namespace): Parsed arguments.
|
|
|
|
Returns:
|
|
int: Process exit status.
|
|
"""
|
|
source: Path = args.source
|
|
if not source.is_dir():
|
|
_emit(f"error: no HTML dump at {source} (is arch-wiki-docs installed?)")
|
|
return 1
|
|
|
|
result = sync_pages(source, args.store / DOCUMENTS_SUBDIR)
|
|
_emit(
|
|
f"converted {result.converted}, changed {result.changed}, "
|
|
f"skipped {result.skipped}",
|
|
)
|
|
|
|
if not args.reindex:
|
|
return 0
|
|
if result.changed == 0:
|
|
_emit("nothing changed; skipping reindex")
|
|
return 0
|
|
|
|
reason = reindex_mod.blocking_reason(args.store, ignore_load=args.ignore_load)
|
|
if reason is not None:
|
|
_emit(f"deferring reindex: {reason}")
|
|
return 0
|
|
|
|
_emit("reindexing...")
|
|
return reindex_mod.run_reindex(args.store)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
"""Parse arguments and dispatch.
|
|
|
|
Parameters:
|
|
argv (list[str] | None): Argument vector, defaulting to ``sys.argv[1:]``.
|
|
|
|
Returns:
|
|
int: Process exit status.
|
|
"""
|
|
args = build_parser().parse_args(argv)
|
|
return run_sync(args)
|