diff --git a/docs/superpowers/contracts/archive-plagiarism-tools-2026-08-18.json b/docs/superpowers/contracts/archive-plagiarism-tools-2026-08-18.json new file mode 100644 index 00000000..daf18ca3 --- /dev/null +++ b/docs/superpowers/contracts/archive-plagiarism-tools-2026-08-18.json @@ -0,0 +1,16 @@ +{ + "title": "Move install_plagiarism_tools.sh and siblings to testsAndMisc-archive", + "objective": "The user said they no longer use the plagiarism-detection tooling (built and just split this session per NEXT_SESSION_PROMPT.md §3). Move it out of testsAndMisc into testsAndMisc-archive with full git history preserved, rather than leaving unused code in the primary repo.", + "acceptance_criteria": [ + "All 13 plagiarism-tool files (install_plagiarism_tools.sh, download_nltk_data.py, check_plagiarism.py, plagiarism_similarity.py, sherlock.py, the 3 lib/plagiarism_*.sh siblings, and the 5 lib/tests/*plagiarism* test files) exist in kuhyx/testsAndMisc-archive with their git history intact, including the pre-rename path (linux_configuration/scripts/utils/install_plagiarism_tools.sh) from before the single_use/periodic_background reorg.", + "The archive push completed and is verified live on GitHub before any deletion happens in the main repo -- deleting first and discovering a failed archive push would lose the only copy.", + "The same 13 files are removed from testsAndMisc via git rm, with no other files touched.", + "check_file_length.sh --all reports the same 14 over-cap files as immediately after the split commit (2a7b9d9) -- this deletion does not change that count, since none of the removed files were over cap.", + "pre-commit run --files passes on the deletion commit's staged set." + ], + "out_of_scope": [ + "Any other unused or academic-tooling files (generate_study_materials.sh, repo_to_study.sh, etc.) -- user explicitly scoped this to just the plagiarism-tool files.", + "Re-running or re-verifying the split work itself; that was already proven correct before the archival decision was made (see install-plagiarism-tools-split-2026-08-18.json)." + ], + "verifier": "gh api repos/kuhyx/testsAndMisc-archive/contents/linux_configuration/scripts/single_use/utils confirming all 13 files present before the git rm commit; git log --oneline -- in testsAndMisc-archive confirming history depth matches the pre-archival testsAndMisc log; check_file_length.sh --all; pre-commit run --files on the deletion commit." +} diff --git a/docs/superpowers/evidence/archive-plagiarism-tools-2026-08-18.json b/docs/superpowers/evidence/archive-plagiarism-tools-2026-08-18.json new file mode 100644 index 00000000..7b0d84fd --- /dev/null +++ b/docs/superpowers/evidence/archive-plagiarism-tools-2026-08-18.json @@ -0,0 +1,43 @@ +{ + "intent": "Move the just-split, unused plagiarism-detection tooling (install_plagiarism_tools.sh and its 12 extracted/test siblings) out of testsAndMisc into testsAndMisc-archive with full git history, per the user's explicit statement that they no longer use it.", + "scope": [ + "linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh, download_nltk_data.py, check_plagiarism.py, plagiarism_similarity.py, sherlock.py (deleted)", + "linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh, plagiarism_sherlock.sh, plagiarism_optional.sh (deleted)", + "linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh, test_plagiarism_python.sh, test_plagiarism_sherlock.sh, test_plagiarism_optional.sh, test_install_plagiarism_tools.sh (deleted)", + "kuhyx/testsAndMisc-archive: same 13 files added, with history", + "Not in scope: any other unused academic-tooling file the user did not name (e.g. generate_study_materials.sh)" + ], + "changes": [ + "Cloned testsAndMisc into a throwaway scratchpad location (never a worktree of the real repo, since git filter-repo rewrites refs and a worktree shares the object store).", + "Ran git filter-repo --force with explicit --path for all 13 current-path files plus the pre-rename path (linux_configuration/scripts/utils/install_plagiarism_tools.sh, from before the single_use/periodic_background reorg) so history predating that rename was not silently dropped.", + "Cloned testsAndMisc-archive separately, added the filtered clone as a remote, and merged it in with --allow-unrelated-histories (required since the two repos share no common root commit).", + "Verified the merge landed all 13 files with git ls-tree, then pushed to testsAndMisc-archive and confirmed live via the GitHub API before touching the main repo.", + "Only after the archive push was confirmed live: git rm the same 13 files from testsAndMisc and committed the deletion.", + "Removed the plag remote from the archive-clone before pushing, so no stray remote-tracking branches from the filtered clone's other refs (copilot-wt, etc.) leaked into the push." + ], + "verification": [ + { + "command": "gh api repos/kuhyx/testsAndMisc-archive/contents/linux_configuration/scripts/single_use/utils", + "result": "pass", + "evidence": "Returned check_plagiarism.py, download_nltk_data.py, install_plagiarism_tools.sh, lib, plagiarism_similarity.py, sherlock.py -- confirmed live on GitHub before the main-repo deletion commit was made." + }, + { + "command": "git log --oneline -- linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh (in the archive clone) vs the same command in testsAndMisc at commit 2a7b9d9", + "result": "pass", + "evidence": "Both show 4 commits with identical messages in the same order (the split, the no-inline-Python gate, the shellcheck fail-build change, the single_use/periodic_background rename) -- history genuinely carried over, not truncated at the rename." + }, + { + "command": "bash meta/scripts/check_file_length.sh --all", + "result": "pass", + "evidence": "14 files over cap, unchanged from immediately after the split commit -- none of the deleted files were over cap, so this operation does not move that number." + } + ], + "risks": [ + "The archive repo has no pre-commit hooks or CI, so nothing re-verifies the moved code still works there -- it is frozen at exactly the state it left testsAndMisc in.", + "git filter-repo was run with --force on a disposable clone rather than a truly fresh `git clone` from the remote; verified this clone's origin was the local repo path (harmless) and that no push was made to it, but noting the deviation from the tool's documented 'fresh clone' expectation." + ], + "rollback": [ + "In testsAndMisc: git revert the deletion commit to restore the 13 files (their content and permissions are unchanged from the split commit, so a straight revert is safe).", + "The testsAndMisc-archive copy can stay regardless of rollback -- having the files in both places briefly is not harmful, just redundant. If it must be undone: git revert the merge commit there, though a merge revert needs the -m flag to pick which parent to keep." + ] +} diff --git a/linux_configuration/scripts/single_use/utils/check_plagiarism.py b/linux_configuration/scripts/single_use/utils/check_plagiarism.py deleted file mode 100755 index 71bb08d0..00000000 --- a/linux_configuration/scripts/single_use/utils/check_plagiarism.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -"""Compare documents for similarity using TF-IDF cosine similarity. - -Extracted from `install_plagiarism_tools.sh`, which used to write this out via -a `cat <<'PYEOF'` heredoc. Inline heredoc Python is invisible to ruff, mypy, -pylint, bandit and pytest, so it lives in its own module instead, matching -`download_nltk_data.py`'s extraction. The comparison engine (file reading, -preprocessing, TF-IDF scoring) lives in `plagiarism_similarity.py`, split out -to keep both files under the 250-line cap. - -The installer copies this file into `$INSTALL_DIR/check_plagiarism.py` and -wires the `plagcheck` wrapper to run it inside the venv. - -Usage: - python check_plagiarism.py file1.txt file2.txt [file3.txt ...] - python check_plagiarism.py --dir /path/to/documents/ -""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import sys - -from plagiarism_similarity import ( - compute_similarity_matrix, - find_similar_passages, - preprocess_text, - read_file, -) - -#: Minimum number of documents needed for a pairwise comparison to make sense. -MIN_DOCUMENTS = 2 - - -def _out(line: str = "") -> None: - sys.stdout.write(line + "\n") - - -def _collect_files(args: argparse.Namespace) -> list[str]: - files: list[str] = list(args.files) - if args.dir: - dir_path = Path(args.dir) - for ext in ("*.txt", "*.pdf", "*.docx", "*.md", "*.tex"): - files.extend(str(f) for f in dir_path.glob(ext)) - return files - - -def _read_documents(files: list[str]) -> tuple[list[str], list[str]]: - documents: list[str] = [] - filenames: list[str] = [] - for f in files: - if not Path(f).exists(): - _out(f"Warning: {f} does not exist") - continue - text = read_file(f) - if text.strip(): - documents.append(preprocess_text(text)) - filenames.append(Path(f).name) - else: - _out(f"Warning: {f} is empty or unreadable") - return documents, filenames - - -def _report_pairs( - documents: list[str], - filenames: list[str], - sim_matrix: list[list[float]], - threshold: float, -) -> list[tuple[int, int, float, str]]: - _out(f"{'Document Pair':<50} {'Similarity':>12}") - _out("-" * 62) - - suspicious_pairs: list[tuple[int, int, float, str]] = [] - for i in range(len(documents)): - for j in range(i + 1, len(documents)): - similarity = sim_matrix[i][j] - pair_name = f"{filenames[i]} <-> {filenames[j]}" - if similarity >= threshold: - suspicious_pairs.append((i, j, similarity, pair_name)) - _out(f"{pair_name:<50} {similarity:>10.1%} ⚠️") - else: - _out(f"{pair_name:<50} {similarity:>10.1%}") - - _out("-" * 62) - return suspicious_pairs - - -def _print_detailed_passages( - files: list[str], suspicious_pairs: list[tuple[int, int, float, str]] -) -> None: - _out("\n" + "=" * 60) - _out(" Detailed Similar Passages") - _out("=" * 60) - - for i, j, sim, pair_name in suspicious_pairs[:3]: - _out(f"\n{pair_name} ({sim:.1%} similar):") - _out("-" * 40) - - raw_docs = [read_file(files[i]), read_file(files[j])] - passages = find_similar_passages(raw_docs[0], raw_docs[1]) - - for s1, s2, psim in passages[:5]: - _out(f'\n[{psim:.0%}] Document 1: "{s1[:100]}..."') - _out(f' Document 2: "{s2[:100]}..."') - - -def main() -> int: - """Compare every pair of input documents and report suspicious pairs.""" - parser = argparse.ArgumentParser( - description="Text Plagiarism Checker - Compare documents for similarity" - ) - parser.add_argument("files", nargs="*", help="Files to compare") - parser.add_argument("--dir", "-d", help="Directory containing documents to compare") - parser.add_argument( - "--threshold", - "-t", - type=float, - default=0.3, - help="Similarity threshold for flagging (0-1, default: 0.3)", - ) - parser.add_argument( - "--detailed", "-v", action="store_true", help="Show detailed similar passages" - ) - - args = parser.parse_args() - - files = _collect_files(args) - if len(files) < MIN_DOCUMENTS: - _out("Error: Need at least 2 files to compare") - parser.print_help() - return 1 - - _out(f"\n{'=' * 60}") - _out(f" Plagiarism Check - Analyzing {len(files)} documents") - _out(f"{'=' * 60}\n") - - documents, filenames = _read_documents(files) - if len(documents) < MIN_DOCUMENTS: - _out("Error: Not enough valid documents to compare") - return 1 - - _out("Computing document similarities...\n") - sim_matrix = compute_similarity_matrix(documents) - - suspicious_pairs = _report_pairs(documents, filenames, sim_matrix, args.threshold) - - if suspicious_pairs: - _out( - f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed " - f"{args.threshold:.0%} similarity threshold\n" - ) - if args.detailed: - _print_detailed_passages(files, suspicious_pairs) - else: - _out(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold") - - _out("\n" + "=" * 60) - _out(" Analysis complete") - _out("=" * 60 + "\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/linux_configuration/scripts/single_use/utils/download_nltk_data.py b/linux_configuration/scripts/single_use/utils/download_nltk_data.py deleted file mode 100755 index 0d3064a5..00000000 --- a/linux_configuration/scripts/single_use/utils/download_nltk_data.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -"""Download the NLTK corpora the plagiarism tools need. - -Extracted from `install_plagiarism_tools.sh`, which used to run this as an -inline `python3 -c` block. Inline Python is invisible to ruff, mypy, pylint, -bandit and pytest, so it lives in its own module instead. - -`nltk` is a dependency of the plagiarism toolchain, not of this repo, so it is -absent from `meta/requirements.txt` on purpose. The installer script pip-installs -it immediately before invoking this module, and probes for it first. -""" - -from __future__ import annotations - -import importlib -import sys - -#: Corpora required by the plagiarism checker. `punkt_tab` is the post-3.8.2 -#: replacement for `punkt`; both are listed so the script works on either. -CORPORA = ( - "punkt", - "stopwords", - "punkt_tab", - "averaged_perceptron_tagger", - "wordnet", -) - - -def main() -> int: - """Download every corpus, reporting any that fail. - - Returns 1 if any download failed, so the calling shell script's `set -e` - sees a real failure instead of a silent partial install. - """ - # Imported dynamically, not at module scope: nltk belongs to the plagiarism - # toolchain rather than this repo, so it is deliberately absent from - # meta/requirements.txt and unresolvable to static analysis. The installer - # pip-installs it immediately before invoking this module. - try: - nltk = importlib.import_module("nltk") - except ImportError: - sys.stderr.write("error: nltk is not installed\n") - return 1 - - failed = [corpus for corpus in CORPORA if not nltk.download(corpus)] - if failed: - sys.stderr.write(f"error: failed to download: {', '.join(failed)}\n") - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh deleted file mode 100755 index 05deda52..00000000 --- a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash -# Install Free & Open Source Plagiarism Detection Tools for Text -# Suitable for academic work (theses, papers, etc.) -# -# Tools installed: -# 1. Python NLP-based similarity detection (sklearn, NLTK, spaCy) -# 2. Sherlock text plagiarism detector -# 3. Ferret (Java-based, if Java available) -# 4. Optional: WCopyfind via Wine (Windows tool) - -set -euo pipefail - -INSTALL_DIR="${HOME}/.local/share/plagiarism-tools" -VENV_DIR="${HOME}/.local/share/plagiarism-venv" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -success() { echo -e "${GREEN}✓ $1${NC}"; } -warn() { echo -e "${YELLOW}⚠ $1${NC}"; } -error() { echo -e "${RED}✗ $1${NC}"; } - -SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -# shellcheck source=lib/plagiarism_python.sh -source "$SCRIPT_DIR/lib/plagiarism_python.sh" -# shellcheck source=lib/plagiarism_sherlock.sh -source "$SCRIPT_DIR/lib/plagiarism_sherlock.sh" -# shellcheck source=lib/plagiarism_optional.sh -source "$SCRIPT_DIR/lib/plagiarism_optional.sh" - -print_summary() { - echo "" - echo "==============================================" - echo " Installation Complete!" - echo "==============================================" - echo "" - echo "Installed tools:" - echo "" - echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)" - echo " Usage: plagcheck file1.txt file2.txt" - echo " plagcheck --dir /path/to/documents/ --detailed" - echo " Location: $INSTALL_DIR/check_plagiarism.py" - echo "" - echo "2. Sherlock (n-gram fingerprinting)" - echo " Location: $INSTALL_DIR/sherlock/sherlock.py" - echo "" - echo "3. Python virtual environment with NLP libraries:" - echo " - scikit-learn (TF-IDF, cosine similarity)" - echo " - NLTK (tokenization, stopwords)" - echo " - spaCy (NLP processing)" - echo " - gensim (document similarity)" - echo " - textdistance, fuzzywuzzy (string matching)" - echo " Activate with: source $VENV_DIR/bin/activate" - echo "" - echo "Quick Start:" - echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed" - echo " plagcheck --dir ./student_papers/ --threshold 0.4" - echo "" - echo "Note: Ensure ~/.local/bin is in your PATH:" - # Printed for the user to paste into their shell rc: $HOME and $PATH must - # stay literal rather than resolve to this machine's values. - # shellcheck disable=SC2016 - echo ' export PATH="$HOME/.local/bin:$PATH"' - echo "" - echo "==============================================" - - if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then - warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:" - # shellcheck disable=SC2016 - echo ' export PATH="$HOME/.local/bin:$PATH"' - fi -} - -main() { - echo "==============================================" - echo " Open Source Plagiarism Detection Installer" - echo " For Academic Text (Theses, Papers, etc.)" - echo "==============================================" - echo "" - - mkdir -p "$INSTALL_DIR" - - install_python_nlp_tools "$SCRIPT_DIR" - install_sherlock "$SCRIPT_DIR" - check_ferret - check_wcopyfind - - print_summary -} - -main "$@" diff --git a/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh b/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh deleted file mode 100755 index d6db2d4f..00000000 --- a/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Stages 3-4: optional plagiarism tools that need a manual download. -# -# Sourced by install_plagiarism_tools.sh; split out to keep it under the -# 250-line cap. Sourced rather than run, so it inherits the caller's strict -# mode and the color/log helpers and INSTALL_DIR global defined above the -# source line. - -# Ferret (Java-based) - Optional -check_ferret() { - echo "" - echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ===" - - if ! command -v java &>/dev/null; then - warn "Java not installed, skipping Ferret" - return - fi - - local ferret_dir="$INSTALL_DIR/ferret" - if [ -d "$ferret_dir" ]; then - return - fi - - mkdir -p "$ferret_dir" - echo "Ferret is a Java-based tool from University of Hertfordshire." - echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/" - echo "Place JAR file in: $ferret_dir" - warn "Ferret requires manual download (academic license)" -} - -# WCopyfind via Wine (Optional) -check_wcopyfind() { - echo "" - echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ===" - - if command -v wine &>/dev/null; then - echo "Wine is available. WCopyfind can be run via Wine." - echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" - echo "Run with: wine /path/to/WCopyfind.exe" - warn "WCopyfind requires manual download" - return - fi - - echo "Wine not installed. To use WCopyfind:" - echo " 1. Install wine: sudo apt install wine (or equivalent)" - echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" - warn "WCopyfind skipped (Wine not available)" -} diff --git a/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh b/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh deleted file mode 100755 index 0eda6da7..00000000 --- a/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# Stage 1: Python NLP-based plagiarism detection environment. -# -# Sourced by install_plagiarism_tools.sh; split out to keep it under the -# 250-line cap. Sourced rather than run, so it inherits the caller's strict -# mode and the color/log helpers and INSTALL_DIR/VENV_DIR globals defined -# above the source line. - -# Create the venv, pip-install the NLP stack, and install check_plagiarism.py -# (plus the plagiarism_similarity.py module it imports) and its "plagcheck" -# wrapper. $1 is the directory install_plagiarism_tools.sh lives in, so both -# scripts can be copied from beside it. -install_python_nlp_tools() { - local script_dir="$1" - - echo "" - echo "=== 1. Installing Python NLP-based Plagiarism Tools ===" - - if ! command -v python3 &>/dev/null; then - error "Python 3 is required but not installed." - exit 1 - fi - - if [ ! -d "$VENV_DIR" ]; then - echo "Creating Python virtual environment..." - python3 -m venv "$VENV_DIR" - success "Virtual environment created at $VENV_DIR" - else - warn "Virtual environment already exists at $VENV_DIR" - fi - - # Created at runtime by `python -m venv` above, so there is nothing on disk - # for the linter to follow at check time. - # shellcheck disable=SC1091 - source "$VENV_DIR/bin/activate" - - echo "Installing Python packages for text similarity detection..." - pip install --upgrade pip - - pip install --progress-bar on \ - scikit-learn \ - nltk \ - spacy \ - gensim \ - numpy \ - pandas \ - python-docx \ - PyPDF2 \ - beautifulsoup4 \ - lxml \ - textdistance \ - fuzzywuzzy \ - python-Levenshtein - - success "Python NLP packages installed" - - echo "Downloading NLTK data (stopwords, punkt tokenizer)..." - python3 "$script_dir/download_nltk_data.py" - success "NLTK data downloaded" - - echo "Downloading spaCy English model..." - python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm" - success "spaCy model installed" - - install -m 755 "$script_dir/check_plagiarism.py" "$INSTALL_DIR/check_plagiarism.py" - install -m 644 "$script_dir/plagiarism_similarity.py" "$INSTALL_DIR/plagiarism_similarity.py" - success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py" - - mkdir -p "$HOME/.local/bin" - cat >"$HOME/.local/bin/plagcheck" </dev/null; then - warn "Git not available, skipping Sherlock installation" - return - fi - - # There are several Sherlock implementations; using a popular Python one - if git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$sherlock_dir" 2>/dev/null; then - success "Sherlock installed at $sherlock_dir" - return - fi - - warn "Could not clone sherlock-py, trying alternative..." - mkdir -p "$sherlock_dir" - install -m 755 "$script_dir/sherlock.py" "$sherlock_dir/sherlock.py" - success "Sherlock installed at $sherlock_dir" -} diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh b/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh deleted file mode 100755 index d57d633c..00000000 --- a/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -# lib/tests/plagiarism_harness.sh — fake python3/pip/git/java/wine behind the -# install_plagiarism_tools.sh split. -# -# Sourced, not executed. Every external tool the installer calls is a shim on -# PATH that records its invocation into $DEV and does the minimum real -# filesystem work the calling code depends on (e.g. `python3 -m venv` must -# leave a `bin/activate` a real `source` can run). INSTALL_DIR, VENV_DIR and -# HOME all point into a throwaway tmpdir so nothing touches the real machine. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -UTILS_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -PASS=0 -FAIL=0 - -_t_pass() { - PASS=$((PASS + 1)) - printf ' OK: %s\n' "$1" -} - -_t_fail() { - FAIL=$((FAIL + 1)) - printf ' FAIL: %s\n' "$1" -} - -_t_eq() { - local want="$1" got="$2" what="$3" - if [[ "$got" == "$want" ]]; then - _t_pass "$what" - else - _t_fail "$what (want '${want}', got '${got}')" - fi -} - -TEST_TMPDIR="$(mktemp -d)" -trap 'rm -rf "${TEST_TMPDIR}"' EXIT - -readonly DEV="${TEST_TMPDIR}/device" -readonly FAKE_BIN="${TEST_TMPDIR}/fake_bin" -mkdir -p "${DEV}" "${FAKE_BIN}" - -# --- fake external tools ---------------------------------------------------- - -cat >"${FAKE_BIN}/python3" <<'PYSHIM' -#!/usr/bin/env bash -set -euo pipefail -DEV="${PLAGIARISM_TEST_DEV}" -printf '%s\n' "python3 $*" >>"${DEV}/calls" -if [[ "$1" == "-m" && "$2" == "venv" ]]; then - venv_dir="$3" - mkdir -p "${venv_dir}/bin" - cat >"${venv_dir}/bin/activate" <<'ACTEOF' -deactivate() { :; } -pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; } -ACTEOF - exit 0 -fi -if [[ "$1" == "-m" && "$2" == "spacy" ]]; then - [[ -f "${DEV}/fail_spacy" ]] && exit 1 - exit 0 -fi -# python3 [args...] — record which script ran and succeed. -printf '%s\n' "ran:$1" >>"${DEV}/scripts_run" -[[ -f "${DEV}/fail_$(basename "$1" .py)" ]] && exit 1 -exit 0 -PYSHIM -chmod +x "${FAKE_BIN}/python3" - -cat >"${FAKE_BIN}/git" <<'GITSHIM' -#!/usr/bin/env bash -set -euo pipefail -DEV="${PLAGIARISM_TEST_DEV}" -printf '%s\n' "git $*" >>"${DEV}/calls" -if [[ "$1" == "clone" ]]; then - [[ -f "${DEV}/fail_clone" ]] && exit 1 - dest="${*: -1}" - mkdir -p "${dest}" - exit 0 -fi -exit 0 -GITSHIM -chmod +x "${FAKE_BIN}/git" - -cat >"${FAKE_BIN}/java" <<'JAVASHIM' -#!/usr/bin/env bash -exit 0 -JAVASHIM -chmod +x "${FAKE_BIN}/java" - -cat >"${FAKE_BIN}/wine" <<'WINESHIM' -#!/usr/bin/env bash -exit 0 -WINESHIM -chmod +x "${FAKE_BIN}/wine" - -export PLAGIARISM_TEST_DEV="${DEV}" -export PATH="${FAKE_BIN}:${PATH}" - -# --- subject under test ----------------------------------------------------- - -HOME="${TEST_TMPDIR}/home" -INSTALL_DIR="${TEST_TMPDIR}/install" -VENV_DIR="${TEST_TMPDIR}/venv" -mkdir -p "${HOME}" "${INSTALL_DIR}" - -success() { :; } -warn() { printf '%s\n' "$*" >>"${DEV}/warnings"; } -error() { printf '%s\n' "$*" >>"${DEV}/errors"; } - -# shellcheck source=../plagiarism_python.sh -. "${UTILS_DIR}/lib/plagiarism_python.sh" -# shellcheck source=../plagiarism_sherlock.sh -. "${UTILS_DIR}/lib/plagiarism_sherlock.sh" -# shellcheck source=../plagiarism_optional.sh -. "${UTILS_DIR}/lib/plagiarism_optional.sh" - -# reset_state — wipe INSTALL_DIR/VENV_DIR/DEV records between test groups so -# each group starts from "nothing installed yet". -reset_state() { - rm -rf "${INSTALL_DIR}" "${VENV_DIR}" "${DEV:?}/calls" "${DEV}/scripts_run" \ - "${DEV}/warnings" "${DEV}/errors" 2>/dev/null || true - mkdir -p "${INSTALL_DIR}" - : >"${DEV}/calls" - : >"${DEV}/scripts_run" - : >"${DEV}/warnings" - : >"${DEV}/errors" -} -reset_state diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh deleted file mode 100755 index a4af8f41..00000000 --- a/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -# End-to-end test for install_plagiarism_tools.sh's main()/print_summary(), -# which stay in the entry script rather than a lib. Runs the real script as a -# subprocess against the same fake python3/pip/git/java/wine as the lib tests, -# with HOME pointed at a throwaway tmpdir so INSTALL_DIR/VENV_DIR (derived -# from $HOME inside the script) land there too. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# shellcheck source=plagiarism_harness.sh -. "${SCRIPT_DIR}/plagiarism_harness.sh" - -readonly ENTRY_SCRIPT="${UTILS_DIR}/install_plagiarism_tools.sh" - -# The entry script derives INSTALL_DIR/VENV_DIR from $HOME itself (a plain -# assignment near its top), independently of the lib tests' INSTALL_DIR/ -# VENV_DIR harness variables. Running it as a real subprocess against the -# harness's $HOME means its paths land under $HOME, not under the harness's -# separate INSTALL_DIR/VENV_DIR -- so assertions below re-derive the same -# paths the script itself computes, rather than reusing the harness ones. -readonly ENTRY_INSTALL_DIR="${HOME}/.local/share/plagiarism-tools" -readonly ENTRY_VENV_DIR="${HOME}/.local/share/plagiarism-venv" - -# --- full run, PATH already in ~/.local/bin ------------------------------- - -mkdir -p "${HOME}/.local/bin" -OUT_FULL="${TEST_TMPDIR}/out_full.log" -PATH="${FAKE_BIN}:${HOME}/.local/bin:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_FULL}" 2>&1 -_status=$? - -if [[ "${_status}" -eq 0 ]]; then - _t_pass "install_plagiarism_tools.sh exits zero on a full successful run" -else - _t_fail "install_plagiarism_tools.sh should exit zero (log: $(cat "${OUT_FULL}"))" -fi - -if grep -q "Open Source Plagiarism Detection Installer" "${OUT_FULL}"; then - _t_pass "main prints the installer banner" -else - _t_fail "main should print the installer banner" -fi - -if grep -q "=== 1. Installing Python NLP-based Plagiarism Tools ===" "${OUT_FULL}"; then - _t_pass "main runs stage 1 (Python NLP tools)" -else - _t_fail "main should run stage 1" -fi - -if grep -q "=== 2. Installing Sherlock Text Plagiarism Detector ===" "${OUT_FULL}"; then - _t_pass "main runs stage 2 (Sherlock)" -else - _t_fail "main should run stage 2" -fi - -if grep -q "=== 3. Checking for Ferret" "${OUT_FULL}"; then - _t_pass "main runs stage 3 (Ferret)" -else - _t_fail "main should run stage 3" -fi - -if grep -q "=== 4. WCopyfind Information" "${OUT_FULL}"; then - _t_pass "main runs stage 4 (WCopyfind)" -else - _t_fail "main should run stage 4" -fi - -if grep -q "Installation Complete!" "${OUT_FULL}"; then - _t_pass "main prints the summary" -else - _t_fail "main should print the summary" -fi - -if grep -q "Location: ${ENTRY_INSTALL_DIR}/check_plagiarism.py" "${OUT_FULL}"; then - _t_pass "the summary reports the checker's installed location" -else - _t_fail "the summary should report the checker's installed location" -fi - -if grep -q "Location: ${ENTRY_INSTALL_DIR}/sherlock/sherlock.py" "${OUT_FULL}"; then - _t_pass "the summary reports sherlock's installed location" -else - _t_fail "the summary should report sherlock's installed location" -fi - -if grep -q "Activate with: source ${ENTRY_VENV_DIR}/bin/activate" "${OUT_FULL}"; then - _t_pass "the summary reports the venv activation command" -else - _t_fail "the summary should report the venv activation command" -fi - -# ~/.local/bin was already on PATH for this run, so the "add it to your PATH" -# nudge must not repeat in the summary. -if grep -q "Add ~/.local/bin to your PATH by adding this" "${OUT_FULL}"; then - _t_fail "the summary should not repeat the PATH nudge when ~/.local/bin is already on PATH" -else - _t_pass "the summary skips the PATH nudge when ~/.local/bin is already on PATH" -fi - -# --- ~/.local/bin missing from PATH: the summary nudges once -------------- - -reset_state -rm -rf "${HOME}/.local" -OUT_NOPATH="${TEST_TMPDIR}/out_nopath.log" -PATH="${FAKE_BIN}:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_NOPATH}" 2>&1 - -if grep -qc "Add ~/.local/bin to your PATH by adding this" "${OUT_NOPATH}"; then - _t_pass "the summary nudges to add ~/.local/bin to PATH when it is missing" -else - _t_fail "the summary should nudge to add ~/.local/bin to PATH when it is missing" -fi - -printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" -[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh deleted file mode 100755 index 87b703ff..00000000 --- a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# Unit tests for lib/plagiarism_optional.sh (Ferret/WCopyfind availability -# checks) against the fake java/wine in plagiarism_harness.sh. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# shellcheck source=plagiarism_harness.sh -. "${SCRIPT_DIR}/plagiarism_harness.sh" - -_no_tool_path() { - local skip="$1" - local dir="${TEST_TMPDIR}/no_${skip}_bin" - mkdir -p "${dir}" - local tool - for tool in python3 git java wine cat chmod mkdir; do - [[ "${tool}" == "${skip}" ]] && continue - [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${dir}/${tool}" - done - printf '%s' "${dir}" -} - -# --- check_ferret: java available ----------------------------------------- - -check_ferret >/tmp/plagiarism_optional_test.log 2>&1 - -if [[ -d "${INSTALL_DIR}/ferret" ]]; then - _t_pass "check_ferret creates the ferret directory when java is available" -else - _t_fail "check_ferret should create ${INSTALL_DIR}/ferret when java is available" -fi - -if grep -q "requires manual download" "${DEV}/warnings"; then - _t_pass "check_ferret explains that Ferret needs a manual download" -else - _t_fail "check_ferret should explain that Ferret needs a manual download" -fi - -# check_ferret creates the directory as a marker; a second run must not -# re-print the manual-download instructions once it already exists. -: >"${DEV}/warnings" -check_ferret >/tmp/plagiarism_optional_test.log 2>&1 -if grep -q "requires manual download" "${DEV}/warnings"; then - _t_fail "check_ferret should not repeat instructions once the ferret dir already exists" -else - _t_pass "check_ferret is a no-op once the ferret directory already exists" -fi - -# --- check_ferret: java unavailable --------------------------------------- - -reset_state -no_java_path="$(_no_tool_path java)" -PATH="${no_java_path}" check_ferret >/dev/null 2>&1 - -if [[ -d "${INSTALL_DIR}/ferret" ]]; then - _t_fail "check_ferret should not create the ferret directory without java" -else - _t_pass "check_ferret skips Ferret entirely when java is unavailable" -fi - -if grep -q "Java not installed" "${DEV}/warnings"; then - _t_pass "check_ferret warns when java is unavailable" -else - _t_fail "check_ferret should warn when java is unavailable" -fi - -# --- check_wcopyfind: wine available --------------------------------------- - -: >"${DEV}/warnings" -check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1 - -if grep -q "Wine is available" /tmp/plagiarism_optional_test.log; then - _t_pass "check_wcopyfind reports Wine as available" -else - _t_fail "check_wcopyfind should report Wine as available" -fi - -if grep -q "requires manual download" "${DEV}/warnings"; then - _t_pass "check_wcopyfind warns that WCopyfind needs a manual download" -else - _t_fail "check_wcopyfind should warn that WCopyfind needs a manual download" -fi - -# --- check_wcopyfind: wine unavailable -------------------------------------- - -: >"${DEV}/warnings" -no_wine_path="$(_no_tool_path wine)" -PATH="${no_wine_path}" check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1 - -if grep -q "Wine not installed" /tmp/plagiarism_optional_test.log; then - _t_pass "check_wcopyfind reports Wine as unavailable" -else - _t_fail "check_wcopyfind should report Wine as unavailable" -fi - -if grep -q "skipped (Wine not available)" "${DEV}/warnings"; then - _t_pass "check_wcopyfind warns that WCopyfind was skipped" -else - _t_fail "check_wcopyfind should warn that WCopyfind was skipped" -fi - -printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" -[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh deleted file mode 100755 index e913e343..00000000 --- a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# Unit tests for lib/plagiarism_python.sh against the fake python3/pip in -# plagiarism_harness.sh. No real network or venv is needed. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# shellcheck source=plagiarism_harness.sh -. "${SCRIPT_DIR}/plagiarism_harness.sh" - -# --- happy path --------------------------------------------------------- - -install_python_nlp_tools "${UTILS_DIR}" >/tmp/plagiarism_python_test.log 2>&1 -_status=$? -if [[ "${_status}" -eq 0 ]]; then - _t_pass "install_python_nlp_tools succeeds against a working toolchain" -else - _t_fail "install_python_nlp_tools should succeed (log: $(cat /tmp/plagiarism_python_test.log))" -fi - -if [[ -d "${VENV_DIR}" ]]; then - _t_pass "install_python_nlp_tools creates the virtual environment" -else - _t_fail "install_python_nlp_tools should create ${VENV_DIR}" -fi - -if grep -q '^python3 -m venv' "${DEV}/calls"; then - _t_pass "install_python_nlp_tools invokes python3 -m venv" -else - _t_fail "install_python_nlp_tools should invoke python3 -m venv" -fi - -if grep -q 'pip install --upgrade pip' "${DEV}/calls"; then - _t_pass "install_python_nlp_tools upgrades pip" -else - _t_fail "install_python_nlp_tools should upgrade pip" -fi - -if grep -q 'scikit-learn' "${DEV}/calls"; then - _t_pass "install_python_nlp_tools installs the NLP package set" -else - _t_fail "install_python_nlp_tools should install scikit-learn and friends" -fi - -if grep -q 'ran:.*download_nltk_data.py' "${DEV}/scripts_run"; then - _t_pass "install_python_nlp_tools runs download_nltk_data.py" -else - _t_fail "install_python_nlp_tools should run download_nltk_data.py" -fi - -if grep -q '^python3 -m spacy download' "${DEV}/calls"; then - _t_pass "install_python_nlp_tools downloads the spaCy model" -else - _t_fail "install_python_nlp_tools should download the spaCy model" -fi - -if [[ -x "${INSTALL_DIR}/check_plagiarism.py" ]]; then - _t_pass "install_python_nlp_tools installs check_plagiarism.py, executable" -else - _t_fail "install_python_nlp_tools should install an executable check_plagiarism.py" -fi - -if [[ -f "${INSTALL_DIR}/plagiarism_similarity.py" ]]; then - _t_pass "install_python_nlp_tools installs plagiarism_similarity.py alongside it" -else - _t_fail "install_python_nlp_tools should install plagiarism_similarity.py -- \ -check_plagiarism.py imports it, so a missing copy breaks at runtime, not install time" -fi - -if [[ -x "${HOME}/.local/bin/plagcheck" ]]; then - _t_pass "install_python_nlp_tools creates an executable plagcheck wrapper" -else - _t_fail "install_python_nlp_tools should create an executable plagcheck wrapper" -fi - -if grep -q "${VENV_DIR}/bin/activate" "${HOME}/.local/bin/plagcheck"; then - _t_pass "plagcheck wrapper activates the venv it was installed against" -else - _t_fail "plagcheck wrapper should reference ${VENV_DIR}/bin/activate" -fi - -if grep -q "${INSTALL_DIR}/check_plagiarism.py" "${HOME}/.local/bin/plagcheck"; then - _t_pass "plagcheck wrapper points at the installed checker script" -else - _t_fail "plagcheck wrapper should point at ${INSTALL_DIR}/check_plagiarism.py" -fi - -# --- venv already exists: warn, don't recreate -------------------------- - -reset_state -mkdir -p "${VENV_DIR}/bin" -cat >"${VENV_DIR}/bin/activate" <<'ACTEOF' -deactivate() { :; } -pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; } -ACTEOF -: >"${DEV}/calls" -install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1 -if grep -q '^python3 -m venv' "${DEV}/calls"; then - _t_fail "install_python_nlp_tools should not recreate an existing venv" -else - _t_pass "install_python_nlp_tools skips venv creation when one already exists" -fi - -# --- python3 missing: fatal, not silently skipped ------------------------ - -reset_state -# A bare `rm -f "${FAKE_BIN}/python3"` does not hide python3 from -# `command -v`: pyenv's shim is still reachable further down the real PATH. -# Instead point PATH at a directory that shadows python3 with nothing before -# falling through to the rest of the fakes, so `command -v python3` fails. -NO_PYTHON_BIN="${TEST_TMPDIR}/no_python_bin" -mkdir -p "${NO_PYTHON_BIN}" -for tool in git java wine cat chmod mkdir; do - [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_PYTHON_BIN}/${tool}" -done -# install_python_nlp_tools calls `exit` (not `return`) on a missing python3, -# by design -- a partial install with no interpreter must abort the whole -# installer, not just this stage. Run it in a subshell so that exit only -# ends the subshell, not this test script. -if (PATH="${NO_PYTHON_BIN}" install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1); then - _t_fail "install_python_nlp_tools should exit non-zero without python3" -else - _t_pass "install_python_nlp_tools exits non-zero when python3 is missing" -fi -if grep -q "Python 3 is required" "${DEV}/errors"; then - _t_pass "install_python_nlp_tools reports the missing-python3 error" -else - _t_fail "install_python_nlp_tools should report the missing-python3 error" -fi - -printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" -[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh deleted file mode 100755 index 5abaa0eb..00000000 --- a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# Unit tests for lib/plagiarism_sherlock.sh against the fake git in -# plagiarism_harness.sh. No real network is needed. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# shellcheck source=plagiarism_harness.sh -. "${SCRIPT_DIR}/plagiarism_harness.sh" - -# --- upstream clone succeeds --------------------------------------------- - -install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 - -if grep -q '^git clone' "${DEV}/calls"; then - _t_pass "install_sherlock clones the upstream sherlock-py tool" -else - _t_fail "install_sherlock should attempt a git clone" -fi - -if [[ -d "${INSTALL_DIR}/sherlock" ]]; then - _t_pass "install_sherlock creates the sherlock directory on a successful clone" -else - _t_fail "install_sherlock should create ${INSTALL_DIR}/sherlock" -fi - -if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then - _t_fail "a successful clone should not also install the bundled fallback sherlock.py" -else - _t_pass "a successful clone does not install the bundled fallback sherlock.py" -fi - -# --- already installed: no-op, warns ------------------------------------- - -: >"${DEV}/calls" -install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 - -if grep -q '^git clone' "${DEV}/calls"; then - _t_fail "install_sherlock should not re-clone when already installed" -else - _t_pass "install_sherlock skips reinstalling when the directory already exists" -fi - -if grep -q "already installed" "${DEV}/warnings"; then - _t_pass "install_sherlock warns that sherlock is already installed" -else - _t_fail "install_sherlock should warn that sherlock is already installed" -fi - -# --- clone fails: falls back to the bundled sherlock.py ------------------- - -reset_state -touch "${DEV}/fail_clone" -install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 - -if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then - _t_pass "install_sherlock falls back to the bundled sherlock.py when the clone fails" -else - _t_fail "install_sherlock should fall back to the bundled sherlock.py when the clone fails" -fi - -if [[ -x "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then - _t_pass "the bundled fallback sherlock.py is installed executable" -else - _t_fail "the bundled fallback sherlock.py should be installed executable" -fi - -if grep -q "Could not clone sherlock-py" "${DEV}/warnings"; then - _t_pass "install_sherlock warns when the clone fails, before falling back" -else - _t_fail "install_sherlock should warn when the clone fails" -fi - -# The installed fallback must be byte-identical to the source file: it is the -# one artifact the git-unavailable path can serve, so a stale copy would ship -# broken plagiarism detection silently. -if diff -q "${UTILS_DIR}/sherlock.py" "${INSTALL_DIR}/sherlock/sherlock.py" >/dev/null; then - _t_pass "the installed fallback sherlock.py matches the source file byte-for-byte" -else - _t_fail "the installed fallback sherlock.py should match ${UTILS_DIR}/sherlock.py" -fi - -# --- git not available: warns, does not attempt to install ---------------- - -reset_state -NO_GIT_BIN="${TEST_TMPDIR}/no_git_bin" -mkdir -p "${NO_GIT_BIN}" -for tool in python3 java wine cat chmod mkdir; do - [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_GIT_BIN}/${tool}" -done -PATH="${NO_GIT_BIN}" install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 - -if [[ -d "${INSTALL_DIR}/sherlock" ]]; then - _t_fail "install_sherlock should not create the sherlock directory without git" -else - _t_pass "install_sherlock skips installation entirely when git is unavailable" -fi - -if grep -q "Git not available" "${DEV}/warnings"; then - _t_pass "install_sherlock warns when git is unavailable" -else - _t_fail "install_sherlock should warn when git is unavailable" -fi - -printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" -[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py b/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py deleted file mode 100755 index b11bee85..00000000 --- a/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -"""Document reading and TF-IDF similarity scoring for check_plagiarism.py. - -Split out of `check_plagiarism.py` to keep it under the 250-line cap: this -module holds the comparison engine (file reading, text preprocessing, and the -TF-IDF/cosine-similarity computations), `check_plagiarism.py` keeps the CLI -and reporting. - -Its third-party imports (numpy, sklearn, nltk) are the plagiarism toolchain's -dependencies, not this repo's, so they are absent from `meta/requirements.txt` -on purpose. Importing them dynamically (rather than with a static `import -nltk`) keeps this module importable -- and lintable -- in an environment -that doesn't have them installed, matching `download_nltk_data.py`'s -extraction. -""" - -from __future__ import annotations - -import importlib -from pathlib import Path -import sys - -#: Cosine similarity above this, for a single sentence pair, is worth reporting -#: as a suspicious passage in --detailed mode. -PASSAGE_SIMILARITY_THRESHOLD = 0.5 - -nltk = importlib.import_module("nltk") -stopwords = importlib.import_module("nltk.corpus").stopwords -nltk_tokenize = importlib.import_module("nltk.tokenize") -sent_tokenize = nltk_tokenize.sent_tokenize -word_tokenize = nltk_tokenize.word_tokenize -TfidfVectorizer = importlib.import_module( - "sklearn.feature_extraction.text" -).TfidfVectorizer -cosine_similarity = importlib.import_module( - "sklearn.metrics.pairwise" -).cosine_similarity - -try: - stopwords.words("english") -except LookupError: - nltk.download("stopwords", quiet=True) - nltk.download("punkt", quiet=True) - - -def _out(line: str = "") -> None: - sys.stdout.write(line + "\n") - - -def _read_pdf(filepath: str) -> str: - try: - pypdf2 = importlib.import_module("PyPDF2") - except ImportError: - _out("Warning: PyPDF2 not installed, cannot read PDF files") - return "" - reader = pypdf2.PdfReader(filepath) - return " ".join(page.extract_text() or "" for page in reader.pages) - - -def _read_docx(filepath: str) -> str: - try: - docx = importlib.import_module("docx") - except ImportError: - _out("Warning: python-docx not installed, cannot read DOCX files") - return "" - doc = docx.Document(filepath) - return " ".join(para.text for para in doc.paragraphs) - - -def read_file(filepath: str) -> str: - """Read text from a .pdf, .docx, or plain-text file.""" - path = Path(filepath) - suffix = path.suffix.lower() - - if suffix == ".pdf": - return _read_pdf(filepath) - if suffix == ".docx": - return _read_docx(filepath) - with path.open(encoding="utf-8", errors="ignore") as f: - return f.read() - - -def preprocess_text(text: str) -> str: - """Lowercase, tokenize, and strip stopwords/non-alphanumerics.""" - text = text.lower() - try: - stop_words = set(stopwords.words("english")) - words = word_tokenize(text) - words = [w for w in words if w.isalnum() and w not in stop_words] - return " ".join(words) - except LookupError: - return " ".join(text.split()) - - -def compute_similarity_matrix(documents: list[str]) -> list[list[float]]: - """Compute the TF-IDF cosine similarity matrix for a set of documents. - - Returns a plain nested list, not the numpy array `cosine_similarity` - produces: numpy is a plagiarism-toolchain dependency the installer - pip-installs at runtime, deliberately absent from `meta/requirements.txt`, - so mypy cannot follow it into this repo's type-checked surface - (`no-any-unimported`). Converting at this boundary keeps every other - signature in the module concretely typed. - """ - vectorizer = TfidfVectorizer(ngram_range=(1, 3), min_df=1, max_df=0.95) - tfidf_matrix = vectorizer.fit_transform(documents) - matrix: list[list[float]] = cosine_similarity(tfidf_matrix).tolist() - return matrix - - -def find_similar_passages( - text1: str, text2: str, min_words: int = 5 -) -> list[tuple[str, str, float]]: - """Find similar sentence-level passages between two texts.""" - sentences1 = [s for s in sent_tokenize(text1) if len(s.split()) >= min_words] - sentences2 = [s for s in sent_tokenize(text2) if len(s.split()) >= min_words] - - if not sentences1 or not sentences2: - return [] - - all_sentences = sentences1 + sentences2 - preprocessed = [preprocess_text(s) for s in all_sentences] - - try: - vectorizer = TfidfVectorizer(ngram_range=(1, 2)) - tfidf_matrix = vectorizer.fit_transform(preprocessed) - except ValueError: - return [] - - n1 = len(sentences1) - similarities = [] - - for i, s1 in enumerate(sentences1): - for j, s2 in enumerate(sentences2): - sim = cosine_similarity( - tfidf_matrix[i : i + 1], tfidf_matrix[n1 + j : n1 + j + 1] - )[0][0] - if sim > PASSAGE_SIMILARITY_THRESHOLD: - similarities.append((s1, s2, sim)) - - return sorted(similarities, key=lambda x: x[2], reverse=True) diff --git a/linux_configuration/scripts/single_use/utils/sherlock.py b/linux_configuration/scripts/single_use/utils/sherlock.py deleted file mode 100755 index 0e9460d6..00000000 --- a/linux_configuration/scripts/single_use/utils/sherlock.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -"""Sherlock - simple text plagiarism detector using n-gram fingerprinting. - -Extracted from `install_plagiarism_tools.sh`'s fallback path, which used to -write this out via a `cat <<'SHERLOCKEOF'` heredoc when cloning the real -sherlock-py failed. Inline heredoc Python is invisible to ruff, mypy, pylint, -bandit and pytest, so it lives in its own module instead, matching -`download_nltk_data.py`'s extraction. - -The installer copies this file into `$SHERLOCK_DIR/sherlock.py` only when -`git clone` of the upstream tool fails. -""" - -from __future__ import annotations - -import argparse -import hashlib -from itertools import combinations -from pathlib import Path -import sys - -#: Minimum number of input files for a pairwise comparison to make sense. -MIN_FILES = 2 - - -def _out(line: str = "") -> None: - sys.stdout.write(line + "\n") - - -def tokenize(text: str) -> list[str]: - """Split text into lowercase alphanumeric words.""" - return [w.lower() for w in text.split() if w.isalnum()] - - -def get_ngrams(tokens: list[str], n: int = 3) -> list[tuple[str, ...]]: - """Generate n-grams from a token list.""" - return [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)] - - -def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set[str]: - """Create a document fingerprint from sampled n-gram hashes.""" - tokens = tokenize(text) - ngrams = get_ngrams(tokens, ngram_size) - - fingerprints = set() - for i, ng in enumerate(ngrams): - if i % sample_rate == 0: - h = hashlib.md5("".join(ng).encode(), usedforsecurity=False).hexdigest()[:8] - fingerprints.add(h) - - return fingerprints - - -def compare_documents(fp1: set[str], fp2: set[str]) -> float: - """Jaccard similarity between two fingerprints.""" - if not fp1 or not fp2: - return 0.0 - intersection = len(fp1 & fp2) - union = len(fp1 | fp2) - return intersection / union if union > 0 else 0.0 - - -def read_document(filepath: str) -> str: - """Read a document's raw text content.""" - with Path(filepath).open(encoding="utf-8", errors="ignore") as f: - return f.read() - - -def main() -> int: - """Fingerprint every input file and report pairwise similarity.""" - parser = argparse.ArgumentParser(description="Sherlock - Text Plagiarism Detector") - parser.add_argument("files", nargs="+", help="Files to compare") - parser.add_argument( - "--ngram", "-n", type=int, default=3, help="N-gram size (default: 3)" - ) - parser.add_argument( - "--threshold", "-t", type=float, default=0.1, help="Similarity threshold" - ) - - args = parser.parse_args() - - if len(args.files) < MIN_FILES: - _out("Need at least 2 files to compare") - return 1 - - docs = {} - for f in args.files: - if Path(f).exists(): - text = read_document(f) - docs[f] = fingerprint(text, args.ngram) - - _out("\nSherlock Plagiarism Analysis") - _out("=" * 50) - - files = list(docs.keys()) - for file1, file2 in combinations(files, 2): - sim = compare_documents(docs[file1], docs[file2]) - name1 = Path(file1).name - name2 = Path(file2).name - flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else "" - _out(f"{name1} <-> {name2}: {sim:.1%}{flag}") - - _out("=" * 50) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())