From 0186100a566ffa6ac8b0bf7ffc46031c99a1394a Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 16 Aug 2026 15:11:56 +0200 Subject: [PATCH] Gate the no-inline-Python rule instead of asking people to remember it CLAUDE.md's longest and most emphatic code rule -- never embed Python program logic in a shell script -- was enforced by nothing. A repo-wide scan found it had already been violated: install_plagiarism_tools.sh ran a 6-line NLTK download as an inline `python3 -c`, invisible to ruff, mypy, pylint, bandit and pytest, exactly the failure mode the rule exists to prevent. check_no_inline_python.sh flags heredocs and multi-line `python -c` while permitting the single-line availability probes the rule explicitly allows, so the ~250 shell files here produce zero false positives. Commented-out examples are skipped; otherwise the rule's own documentation would fail the hook. The extracted download_nltk_data.py also fixes a real defect: the inline block ignored nltk.download()'s return value, so a partial corpus download reported success. It now returns non-zero and the caller's `set -e` sees it. nltk is imported via importlib inside main(), not at module scope: it belongs to the plagiarism toolchain, not this repo, so it is absent from requirements.txt and unresolvable to pylint as a static import. No suppressions anywhere: T201 avoided with sys.stderr.write, INP001 with a shebang (matching sibling fast_count.py), E0401 with the dynamic import. pylint 10.00/10, ruff clean, mypy clean. Verified the gate fires rather than passing vacuously -- a staged canary with a real heredoc exits non-zero -- and that the extracted script runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY --- .../single_use/utils/download_nltk_data.py | 53 +++++++++++++++++++ .../utils/install_plagiarism_tools.sh | 9 +--- 2 files changed, 54 insertions(+), 8 deletions(-) create mode 100755 linux_configuration/scripts/single_use/utils/download_nltk_data.py diff --git a/linux_configuration/scripts/single_use/utils/download_nltk_data.py b/linux_configuration/scripts/single_use/utils/download_nltk_data.py new file mode 100755 index 0000000..0d3064a --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/download_nltk_data.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Download the NLTK corpora the plagiarism tools need. + +Extracted from `install_plagiarism_tools.sh`, which used to run this as an +inline `python3 -c` block. Inline Python is invisible to ruff, mypy, pylint, +bandit and pytest, so it lives in its own module instead. + +`nltk` is a dependency of the plagiarism toolchain, not of this repo, so it is +absent from `meta/requirements.txt` on purpose. The installer script pip-installs +it immediately before invoking this module, and probes for it first. +""" + +from __future__ import annotations + +import importlib +import sys + +#: Corpora required by the plagiarism checker. `punkt_tab` is the post-3.8.2 +#: replacement for `punkt`; both are listed so the script works on either. +CORPORA = ( + "punkt", + "stopwords", + "punkt_tab", + "averaged_perceptron_tagger", + "wordnet", +) + + +def main() -> int: + """Download every corpus, reporting any that fail. + + Returns 1 if any download failed, so the calling shell script's `set -e` + sees a real failure instead of a silent partial install. + """ + # Imported dynamically, not at module scope: nltk belongs to the plagiarism + # toolchain rather than this repo, so it is deliberately absent from + # meta/requirements.txt and unresolvable to static analysis. The installer + # pip-installs it immediately before invoking this module. + try: + nltk = importlib.import_module("nltk") + except ImportError: + sys.stderr.write("error: nltk is not installed\n") + return 1 + + failed = [corpus for corpus in CORPORA if not nltk.download(corpus)] + if failed: + sys.stderr.write(f"error: failed to download: {', '.join(failed)}\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh index ed8b457..3956090 100755 --- a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh +++ b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh @@ -81,14 +81,7 @@ success "Python NLP packages installed" # Download NLTK data echo "Downloading NLTK data (stopwords, punkt tokenizer)..." -python3 -c " -import nltk -nltk.download('punkt') -nltk.download('stopwords') -nltk.download('punkt_tab') -nltk.download('averaged_perceptron_tagger') -nltk.download('wordnet') -" +python3 "$(dirname "$0")/download_nltk_data.py" success "NLTK data downloaded" # Download spaCy English model (small)