diff --git a/linux_configuration/scripts/single_use/utils/check_plagiarism.py b/linux_configuration/scripts/single_use/utils/check_plagiarism.py new file mode 100755 index 0000000..71bb08d --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/check_plagiarism.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Compare documents for similarity using TF-IDF cosine similarity. + +Extracted from `install_plagiarism_tools.sh`, which used to write this out via +a `cat <<'PYEOF'` heredoc. Inline heredoc Python is invisible to ruff, mypy, +pylint, bandit and pytest, so it lives in its own module instead, matching +`download_nltk_data.py`'s extraction. The comparison engine (file reading, +preprocessing, TF-IDF scoring) lives in `plagiarism_similarity.py`, split out +to keep both files under the 250-line cap. + +The installer copies this file into `$INSTALL_DIR/check_plagiarism.py` and +wires the `plagcheck` wrapper to run it inside the venv. + +Usage: + python check_plagiarism.py file1.txt file2.txt [file3.txt ...] + python check_plagiarism.py --dir /path/to/documents/ +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +from plagiarism_similarity import ( + compute_similarity_matrix, + find_similar_passages, + preprocess_text, + read_file, +) + +#: Minimum number of documents needed for a pairwise comparison to make sense. +MIN_DOCUMENTS = 2 + + +def _out(line: str = "") -> None: + sys.stdout.write(line + "\n") + + +def _collect_files(args: argparse.Namespace) -> list[str]: + files: list[str] = list(args.files) + if args.dir: + dir_path = Path(args.dir) + for ext in ("*.txt", "*.pdf", "*.docx", "*.md", "*.tex"): + files.extend(str(f) for f in dir_path.glob(ext)) + return files + + +def _read_documents(files: list[str]) -> tuple[list[str], list[str]]: + documents: list[str] = [] + filenames: list[str] = [] + for f in files: + if not Path(f).exists(): + _out(f"Warning: {f} does not exist") + continue + text = read_file(f) + if text.strip(): + documents.append(preprocess_text(text)) + filenames.append(Path(f).name) + else: + _out(f"Warning: {f} is empty or unreadable") + return documents, filenames + + +def _report_pairs( + documents: list[str], + filenames: list[str], + sim_matrix: list[list[float]], + threshold: float, +) -> list[tuple[int, int, float, str]]: + _out(f"{'Document Pair':<50} {'Similarity':>12}") + _out("-" * 62) + + suspicious_pairs: list[tuple[int, int, float, str]] = [] + for i in range(len(documents)): + for j in range(i + 1, len(documents)): + similarity = sim_matrix[i][j] + pair_name = f"{filenames[i]} <-> {filenames[j]}" + if similarity >= threshold: + suspicious_pairs.append((i, j, similarity, pair_name)) + _out(f"{pair_name:<50} {similarity:>10.1%} ⚠️") + else: + _out(f"{pair_name:<50} {similarity:>10.1%}") + + _out("-" * 62) + return suspicious_pairs + + +def _print_detailed_passages( + files: list[str], suspicious_pairs: list[tuple[int, int, float, str]] +) -> None: + _out("\n" + "=" * 60) + _out(" Detailed Similar Passages") + _out("=" * 60) + + for i, j, sim, pair_name in suspicious_pairs[:3]: + _out(f"\n{pair_name} ({sim:.1%} similar):") + _out("-" * 40) + + raw_docs = [read_file(files[i]), read_file(files[j])] + passages = find_similar_passages(raw_docs[0], raw_docs[1]) + + for s1, s2, psim in passages[:5]: + _out(f'\n[{psim:.0%}] Document 1: "{s1[:100]}..."') + _out(f' Document 2: "{s2[:100]}..."') + + +def main() -> int: + """Compare every pair of input documents and report suspicious pairs.""" + parser = argparse.ArgumentParser( + description="Text Plagiarism Checker - Compare documents for similarity" + ) + parser.add_argument("files", nargs="*", help="Files to compare") + parser.add_argument("--dir", "-d", help="Directory containing documents to compare") + parser.add_argument( + "--threshold", + "-t", + type=float, + default=0.3, + help="Similarity threshold for flagging (0-1, default: 0.3)", + ) + parser.add_argument( + "--detailed", "-v", action="store_true", help="Show detailed similar passages" + ) + + args = parser.parse_args() + + files = _collect_files(args) + if len(files) < MIN_DOCUMENTS: + _out("Error: Need at least 2 files to compare") + parser.print_help() + return 1 + + _out(f"\n{'=' * 60}") + _out(f" Plagiarism Check - Analyzing {len(files)} documents") + _out(f"{'=' * 60}\n") + + documents, filenames = _read_documents(files) + if len(documents) < MIN_DOCUMENTS: + _out("Error: Not enough valid documents to compare") + return 1 + + _out("Computing document similarities...\n") + sim_matrix = compute_similarity_matrix(documents) + + suspicious_pairs = _report_pairs(documents, filenames, sim_matrix, args.threshold) + + if suspicious_pairs: + _out( + f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed " + f"{args.threshold:.0%} similarity threshold\n" + ) + if args.detailed: + _print_detailed_passages(files, suspicious_pairs) + else: + _out(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold") + + _out("\n" + "=" * 60) + _out(" Analysis complete") + _out("=" * 60 + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/linux_configuration/scripts/single_use/utils/download_nltk_data.py b/linux_configuration/scripts/single_use/utils/download_nltk_data.py new file mode 100755 index 0000000..0d3064a --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/download_nltk_data.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Download the NLTK corpora the plagiarism tools need. + +Extracted from `install_plagiarism_tools.sh`, which used to run this as an +inline `python3 -c` block. Inline Python is invisible to ruff, mypy, pylint, +bandit and pytest, so it lives in its own module instead. + +`nltk` is a dependency of the plagiarism toolchain, not of this repo, so it is +absent from `meta/requirements.txt` on purpose. The installer script pip-installs +it immediately before invoking this module, and probes for it first. +""" + +from __future__ import annotations + +import importlib +import sys + +#: Corpora required by the plagiarism checker. `punkt_tab` is the post-3.8.2 +#: replacement for `punkt`; both are listed so the script works on either. +CORPORA = ( + "punkt", + "stopwords", + "punkt_tab", + "averaged_perceptron_tagger", + "wordnet", +) + + +def main() -> int: + """Download every corpus, reporting any that fail. + + Returns 1 if any download failed, so the calling shell script's `set -e` + sees a real failure instead of a silent partial install. + """ + # Imported dynamically, not at module scope: nltk belongs to the plagiarism + # toolchain rather than this repo, so it is deliberately absent from + # meta/requirements.txt and unresolvable to static analysis. The installer + # pip-installs it immediately before invoking this module. + try: + nltk = importlib.import_module("nltk") + except ImportError: + sys.stderr.write("error: nltk is not installed\n") + return 1 + + failed = [corpus for corpus in CORPORA if not nltk.download(corpus)] + if failed: + sys.stderr.write(f"error: failed to download: {', '.join(failed)}\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh new file mode 100755 index 0000000..05deda5 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Install Free & Open Source Plagiarism Detection Tools for Text +# Suitable for academic work (theses, papers, etc.) +# +# Tools installed: +# 1. Python NLP-based similarity detection (sklearn, NLTK, spaCy) +# 2. Sherlock text plagiarism detector +# 3. Ferret (Java-based, if Java available) +# 4. Optional: WCopyfind via Wine (Windows tool) + +set -euo pipefail + +INSTALL_DIR="${HOME}/.local/share/plagiarism-tools" +VENV_DIR="${HOME}/.local/share/plagiarism-venv" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +success() { echo -e "${GREEN}✓ $1${NC}"; } +warn() { echo -e "${YELLOW}⚠ $1${NC}"; } +error() { echo -e "${RED}✗ $1${NC}"; } + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +# shellcheck source=lib/plagiarism_python.sh +source "$SCRIPT_DIR/lib/plagiarism_python.sh" +# shellcheck source=lib/plagiarism_sherlock.sh +source "$SCRIPT_DIR/lib/plagiarism_sherlock.sh" +# shellcheck source=lib/plagiarism_optional.sh +source "$SCRIPT_DIR/lib/plagiarism_optional.sh" + +print_summary() { + echo "" + echo "==============================================" + echo " Installation Complete!" + echo "==============================================" + echo "" + echo "Installed tools:" + echo "" + echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)" + echo " Usage: plagcheck file1.txt file2.txt" + echo " plagcheck --dir /path/to/documents/ --detailed" + echo " Location: $INSTALL_DIR/check_plagiarism.py" + echo "" + echo "2. Sherlock (n-gram fingerprinting)" + echo " Location: $INSTALL_DIR/sherlock/sherlock.py" + echo "" + echo "3. Python virtual environment with NLP libraries:" + echo " - scikit-learn (TF-IDF, cosine similarity)" + echo " - NLTK (tokenization, stopwords)" + echo " - spaCy (NLP processing)" + echo " - gensim (document similarity)" + echo " - textdistance, fuzzywuzzy (string matching)" + echo " Activate with: source $VENV_DIR/bin/activate" + echo "" + echo "Quick Start:" + echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed" + echo " plagcheck --dir ./student_papers/ --threshold 0.4" + echo "" + echo "Note: Ensure ~/.local/bin is in your PATH:" + # Printed for the user to paste into their shell rc: $HOME and $PATH must + # stay literal rather than resolve to this machine's values. + # shellcheck disable=SC2016 + echo ' export PATH="$HOME/.local/bin:$PATH"' + echo "" + echo "==============================================" + + if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:" + # shellcheck disable=SC2016 + echo ' export PATH="$HOME/.local/bin:$PATH"' + fi +} + +main() { + echo "==============================================" + echo " Open Source Plagiarism Detection Installer" + echo " For Academic Text (Theses, Papers, etc.)" + echo "==============================================" + echo "" + + mkdir -p "$INSTALL_DIR" + + install_python_nlp_tools "$SCRIPT_DIR" + install_sherlock "$SCRIPT_DIR" + check_ferret + check_wcopyfind + + print_summary +} + +main "$@" diff --git a/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh b/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh new file mode 100755 index 0000000..d6db2d4 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Stages 3-4: optional plagiarism tools that need a manual download. +# +# Sourced by install_plagiarism_tools.sh; split out to keep it under the +# 250-line cap. Sourced rather than run, so it inherits the caller's strict +# mode and the color/log helpers and INSTALL_DIR global defined above the +# source line. + +# Ferret (Java-based) - Optional +check_ferret() { + echo "" + echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ===" + + if ! command -v java &>/dev/null; then + warn "Java not installed, skipping Ferret" + return + fi + + local ferret_dir="$INSTALL_DIR/ferret" + if [ -d "$ferret_dir" ]; then + return + fi + + mkdir -p "$ferret_dir" + echo "Ferret is a Java-based tool from University of Hertfordshire." + echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/" + echo "Place JAR file in: $ferret_dir" + warn "Ferret requires manual download (academic license)" +} + +# WCopyfind via Wine (Optional) +check_wcopyfind() { + echo "" + echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ===" + + if command -v wine &>/dev/null; then + echo "Wine is available. WCopyfind can be run via Wine." + echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + echo "Run with: wine /path/to/WCopyfind.exe" + warn "WCopyfind requires manual download" + return + fi + + echo "Wine not installed. To use WCopyfind:" + echo " 1. Install wine: sudo apt install wine (or equivalent)" + echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + warn "WCopyfind skipped (Wine not available)" +} diff --git a/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh b/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh new file mode 100755 index 0000000..0eda6da --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Stage 1: Python NLP-based plagiarism detection environment. +# +# Sourced by install_plagiarism_tools.sh; split out to keep it under the +# 250-line cap. Sourced rather than run, so it inherits the caller's strict +# mode and the color/log helpers and INSTALL_DIR/VENV_DIR globals defined +# above the source line. + +# Create the venv, pip-install the NLP stack, and install check_plagiarism.py +# (plus the plagiarism_similarity.py module it imports) and its "plagcheck" +# wrapper. $1 is the directory install_plagiarism_tools.sh lives in, so both +# scripts can be copied from beside it. +install_python_nlp_tools() { + local script_dir="$1" + + echo "" + echo "=== 1. Installing Python NLP-based Plagiarism Tools ===" + + if ! command -v python3 &>/dev/null; then + error "Python 3 is required but not installed." + exit 1 + fi + + if [ ! -d "$VENV_DIR" ]; then + echo "Creating Python virtual environment..." + python3 -m venv "$VENV_DIR" + success "Virtual environment created at $VENV_DIR" + else + warn "Virtual environment already exists at $VENV_DIR" + fi + + # Created at runtime by `python -m venv` above, so there is nothing on disk + # for the linter to follow at check time. + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + + echo "Installing Python packages for text similarity detection..." + pip install --upgrade pip + + pip install --progress-bar on \ + scikit-learn \ + nltk \ + spacy \ + gensim \ + numpy \ + pandas \ + python-docx \ + PyPDF2 \ + beautifulsoup4 \ + lxml \ + textdistance \ + fuzzywuzzy \ + python-Levenshtein + + success "Python NLP packages installed" + + echo "Downloading NLTK data (stopwords, punkt tokenizer)..." + python3 "$script_dir/download_nltk_data.py" + success "NLTK data downloaded" + + echo "Downloading spaCy English model..." + python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm" + success "spaCy model installed" + + install -m 755 "$script_dir/check_plagiarism.py" "$INSTALL_DIR/check_plagiarism.py" + install -m 644 "$script_dir/plagiarism_similarity.py" "$INSTALL_DIR/plagiarism_similarity.py" + success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py" + + mkdir -p "$HOME/.local/bin" + cat >"$HOME/.local/bin/plagcheck" </dev/null; then + warn "Git not available, skipping Sherlock installation" + return + fi + + # There are several Sherlock implementations; using a popular Python one + if git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$sherlock_dir" 2>/dev/null; then + success "Sherlock installed at $sherlock_dir" + return + fi + + warn "Could not clone sherlock-py, trying alternative..." + mkdir -p "$sherlock_dir" + install -m 755 "$script_dir/sherlock.py" "$sherlock_dir/sherlock.py" + success "Sherlock installed at $sherlock_dir" +} diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh b/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh new file mode 100755 index 0000000..d57d633 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# lib/tests/plagiarism_harness.sh — fake python3/pip/git/java/wine behind the +# install_plagiarism_tools.sh split. +# +# Sourced, not executed. Every external tool the installer calls is a shim on +# PATH that records its invocation into $DEV and does the minimum real +# filesystem work the calling code depends on (e.g. `python3 -m venv` must +# leave a `bin/activate` a real `source` can run). INSTALL_DIR, VENV_DIR and +# HOME all point into a throwaway tmpdir so nothing touches the real machine. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +UTILS_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +PASS=0 +FAIL=0 + +_t_pass() { + PASS=$((PASS + 1)) + printf ' OK: %s\n' "$1" +} + +_t_fail() { + FAIL=$((FAIL + 1)) + printf ' FAIL: %s\n' "$1" +} + +_t_eq() { + local want="$1" got="$2" what="$3" + if [[ "$got" == "$want" ]]; then + _t_pass "$what" + else + _t_fail "$what (want '${want}', got '${got}')" + fi +} + +TEST_TMPDIR="$(mktemp -d)" +trap 'rm -rf "${TEST_TMPDIR}"' EXIT + +readonly DEV="${TEST_TMPDIR}/device" +readonly FAKE_BIN="${TEST_TMPDIR}/fake_bin" +mkdir -p "${DEV}" "${FAKE_BIN}" + +# --- fake external tools ---------------------------------------------------- + +cat >"${FAKE_BIN}/python3" <<'PYSHIM' +#!/usr/bin/env bash +set -euo pipefail +DEV="${PLAGIARISM_TEST_DEV}" +printf '%s\n' "python3 $*" >>"${DEV}/calls" +if [[ "$1" == "-m" && "$2" == "venv" ]]; then + venv_dir="$3" + mkdir -p "${venv_dir}/bin" + cat >"${venv_dir}/bin/activate" <<'ACTEOF' +deactivate() { :; } +pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; } +ACTEOF + exit 0 +fi +if [[ "$1" == "-m" && "$2" == "spacy" ]]; then + [[ -f "${DEV}/fail_spacy" ]] && exit 1 + exit 0 +fi +# python3 [args...] — record which script ran and succeed. +printf '%s\n' "ran:$1" >>"${DEV}/scripts_run" +[[ -f "${DEV}/fail_$(basename "$1" .py)" ]] && exit 1 +exit 0 +PYSHIM +chmod +x "${FAKE_BIN}/python3" + +cat >"${FAKE_BIN}/git" <<'GITSHIM' +#!/usr/bin/env bash +set -euo pipefail +DEV="${PLAGIARISM_TEST_DEV}" +printf '%s\n' "git $*" >>"${DEV}/calls" +if [[ "$1" == "clone" ]]; then + [[ -f "${DEV}/fail_clone" ]] && exit 1 + dest="${*: -1}" + mkdir -p "${dest}" + exit 0 +fi +exit 0 +GITSHIM +chmod +x "${FAKE_BIN}/git" + +cat >"${FAKE_BIN}/java" <<'JAVASHIM' +#!/usr/bin/env bash +exit 0 +JAVASHIM +chmod +x "${FAKE_BIN}/java" + +cat >"${FAKE_BIN}/wine" <<'WINESHIM' +#!/usr/bin/env bash +exit 0 +WINESHIM +chmod +x "${FAKE_BIN}/wine" + +export PLAGIARISM_TEST_DEV="${DEV}" +export PATH="${FAKE_BIN}:${PATH}" + +# --- subject under test ----------------------------------------------------- + +HOME="${TEST_TMPDIR}/home" +INSTALL_DIR="${TEST_TMPDIR}/install" +VENV_DIR="${TEST_TMPDIR}/venv" +mkdir -p "${HOME}" "${INSTALL_DIR}" + +success() { :; } +warn() { printf '%s\n' "$*" >>"${DEV}/warnings"; } +error() { printf '%s\n' "$*" >>"${DEV}/errors"; } + +# shellcheck source=../plagiarism_python.sh +. "${UTILS_DIR}/lib/plagiarism_python.sh" +# shellcheck source=../plagiarism_sherlock.sh +. "${UTILS_DIR}/lib/plagiarism_sherlock.sh" +# shellcheck source=../plagiarism_optional.sh +. "${UTILS_DIR}/lib/plagiarism_optional.sh" + +# reset_state — wipe INSTALL_DIR/VENV_DIR/DEV records between test groups so +# each group starts from "nothing installed yet". +reset_state() { + rm -rf "${INSTALL_DIR}" "${VENV_DIR}" "${DEV:?}/calls" "${DEV}/scripts_run" \ + "${DEV}/warnings" "${DEV}/errors" 2>/dev/null || true + mkdir -p "${INSTALL_DIR}" + : >"${DEV}/calls" + : >"${DEV}/scripts_run" + : >"${DEV}/warnings" + : >"${DEV}/errors" +} +reset_state diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh new file mode 100755 index 0000000..a4af8f4 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# End-to-end test for install_plagiarism_tools.sh's main()/print_summary(), +# which stay in the entry script rather than a lib. Runs the real script as a +# subprocess against the same fake python3/pip/git/java/wine as the lib tests, +# with HOME pointed at a throwaway tmpdir so INSTALL_DIR/VENV_DIR (derived +# from $HOME inside the script) land there too. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +readonly ENTRY_SCRIPT="${UTILS_DIR}/install_plagiarism_tools.sh" + +# The entry script derives INSTALL_DIR/VENV_DIR from $HOME itself (a plain +# assignment near its top), independently of the lib tests' INSTALL_DIR/ +# VENV_DIR harness variables. Running it as a real subprocess against the +# harness's $HOME means its paths land under $HOME, not under the harness's +# separate INSTALL_DIR/VENV_DIR -- so assertions below re-derive the same +# paths the script itself computes, rather than reusing the harness ones. +readonly ENTRY_INSTALL_DIR="${HOME}/.local/share/plagiarism-tools" +readonly ENTRY_VENV_DIR="${HOME}/.local/share/plagiarism-venv" + +# --- full run, PATH already in ~/.local/bin ------------------------------- + +mkdir -p "${HOME}/.local/bin" +OUT_FULL="${TEST_TMPDIR}/out_full.log" +PATH="${FAKE_BIN}:${HOME}/.local/bin:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_FULL}" 2>&1 +_status=$? + +if [[ "${_status}" -eq 0 ]]; then + _t_pass "install_plagiarism_tools.sh exits zero on a full successful run" +else + _t_fail "install_plagiarism_tools.sh should exit zero (log: $(cat "${OUT_FULL}"))" +fi + +if grep -q "Open Source Plagiarism Detection Installer" "${OUT_FULL}"; then + _t_pass "main prints the installer banner" +else + _t_fail "main should print the installer banner" +fi + +if grep -q "=== 1. Installing Python NLP-based Plagiarism Tools ===" "${OUT_FULL}"; then + _t_pass "main runs stage 1 (Python NLP tools)" +else + _t_fail "main should run stage 1" +fi + +if grep -q "=== 2. Installing Sherlock Text Plagiarism Detector ===" "${OUT_FULL}"; then + _t_pass "main runs stage 2 (Sherlock)" +else + _t_fail "main should run stage 2" +fi + +if grep -q "=== 3. Checking for Ferret" "${OUT_FULL}"; then + _t_pass "main runs stage 3 (Ferret)" +else + _t_fail "main should run stage 3" +fi + +if grep -q "=== 4. WCopyfind Information" "${OUT_FULL}"; then + _t_pass "main runs stage 4 (WCopyfind)" +else + _t_fail "main should run stage 4" +fi + +if grep -q "Installation Complete!" "${OUT_FULL}"; then + _t_pass "main prints the summary" +else + _t_fail "main should print the summary" +fi + +if grep -q "Location: ${ENTRY_INSTALL_DIR}/check_plagiarism.py" "${OUT_FULL}"; then + _t_pass "the summary reports the checker's installed location" +else + _t_fail "the summary should report the checker's installed location" +fi + +if grep -q "Location: ${ENTRY_INSTALL_DIR}/sherlock/sherlock.py" "${OUT_FULL}"; then + _t_pass "the summary reports sherlock's installed location" +else + _t_fail "the summary should report sherlock's installed location" +fi + +if grep -q "Activate with: source ${ENTRY_VENV_DIR}/bin/activate" "${OUT_FULL}"; then + _t_pass "the summary reports the venv activation command" +else + _t_fail "the summary should report the venv activation command" +fi + +# ~/.local/bin was already on PATH for this run, so the "add it to your PATH" +# nudge must not repeat in the summary. +if grep -q "Add ~/.local/bin to your PATH by adding this" "${OUT_FULL}"; then + _t_fail "the summary should not repeat the PATH nudge when ~/.local/bin is already on PATH" +else + _t_pass "the summary skips the PATH nudge when ~/.local/bin is already on PATH" +fi + +# --- ~/.local/bin missing from PATH: the summary nudges once -------------- + +reset_state +rm -rf "${HOME}/.local" +OUT_NOPATH="${TEST_TMPDIR}/out_nopath.log" +PATH="${FAKE_BIN}:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_NOPATH}" 2>&1 + +if grep -qc "Add ~/.local/bin to your PATH by adding this" "${OUT_NOPATH}"; then + _t_pass "the summary nudges to add ~/.local/bin to PATH when it is missing" +else + _t_fail "the summary should nudge to add ~/.local/bin to PATH when it is missing" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh new file mode 100755 index 0000000..87b703f --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Unit tests for lib/plagiarism_optional.sh (Ferret/WCopyfind availability +# checks) against the fake java/wine in plagiarism_harness.sh. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +_no_tool_path() { + local skip="$1" + local dir="${TEST_TMPDIR}/no_${skip}_bin" + mkdir -p "${dir}" + local tool + for tool in python3 git java wine cat chmod mkdir; do + [[ "${tool}" == "${skip}" ]] && continue + [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${dir}/${tool}" + done + printf '%s' "${dir}" +} + +# --- check_ferret: java available ----------------------------------------- + +check_ferret >/tmp/plagiarism_optional_test.log 2>&1 + +if [[ -d "${INSTALL_DIR}/ferret" ]]; then + _t_pass "check_ferret creates the ferret directory when java is available" +else + _t_fail "check_ferret should create ${INSTALL_DIR}/ferret when java is available" +fi + +if grep -q "requires manual download" "${DEV}/warnings"; then + _t_pass "check_ferret explains that Ferret needs a manual download" +else + _t_fail "check_ferret should explain that Ferret needs a manual download" +fi + +# check_ferret creates the directory as a marker; a second run must not +# re-print the manual-download instructions once it already exists. +: >"${DEV}/warnings" +check_ferret >/tmp/plagiarism_optional_test.log 2>&1 +if grep -q "requires manual download" "${DEV}/warnings"; then + _t_fail "check_ferret should not repeat instructions once the ferret dir already exists" +else + _t_pass "check_ferret is a no-op once the ferret directory already exists" +fi + +# --- check_ferret: java unavailable --------------------------------------- + +reset_state +no_java_path="$(_no_tool_path java)" +PATH="${no_java_path}" check_ferret >/dev/null 2>&1 + +if [[ -d "${INSTALL_DIR}/ferret" ]]; then + _t_fail "check_ferret should not create the ferret directory without java" +else + _t_pass "check_ferret skips Ferret entirely when java is unavailable" +fi + +if grep -q "Java not installed" "${DEV}/warnings"; then + _t_pass "check_ferret warns when java is unavailable" +else + _t_fail "check_ferret should warn when java is unavailable" +fi + +# --- check_wcopyfind: wine available --------------------------------------- + +: >"${DEV}/warnings" +check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1 + +if grep -q "Wine is available" /tmp/plagiarism_optional_test.log; then + _t_pass "check_wcopyfind reports Wine as available" +else + _t_fail "check_wcopyfind should report Wine as available" +fi + +if grep -q "requires manual download" "${DEV}/warnings"; then + _t_pass "check_wcopyfind warns that WCopyfind needs a manual download" +else + _t_fail "check_wcopyfind should warn that WCopyfind needs a manual download" +fi + +# --- check_wcopyfind: wine unavailable -------------------------------------- + +: >"${DEV}/warnings" +no_wine_path="$(_no_tool_path wine)" +PATH="${no_wine_path}" check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1 + +if grep -q "Wine not installed" /tmp/plagiarism_optional_test.log; then + _t_pass "check_wcopyfind reports Wine as unavailable" +else + _t_fail "check_wcopyfind should report Wine as unavailable" +fi + +if grep -q "skipped (Wine not available)" "${DEV}/warnings"; then + _t_pass "check_wcopyfind warns that WCopyfind was skipped" +else + _t_fail "check_wcopyfind should warn that WCopyfind was skipped" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh new file mode 100755 index 0000000..e913e34 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Unit tests for lib/plagiarism_python.sh against the fake python3/pip in +# plagiarism_harness.sh. No real network or venv is needed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +# --- happy path --------------------------------------------------------- + +install_python_nlp_tools "${UTILS_DIR}" >/tmp/plagiarism_python_test.log 2>&1 +_status=$? +if [[ "${_status}" -eq 0 ]]; then + _t_pass "install_python_nlp_tools succeeds against a working toolchain" +else + _t_fail "install_python_nlp_tools should succeed (log: $(cat /tmp/plagiarism_python_test.log))" +fi + +if [[ -d "${VENV_DIR}" ]]; then + _t_pass "install_python_nlp_tools creates the virtual environment" +else + _t_fail "install_python_nlp_tools should create ${VENV_DIR}" +fi + +if grep -q '^python3 -m venv' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools invokes python3 -m venv" +else + _t_fail "install_python_nlp_tools should invoke python3 -m venv" +fi + +if grep -q 'pip install --upgrade pip' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools upgrades pip" +else + _t_fail "install_python_nlp_tools should upgrade pip" +fi + +if grep -q 'scikit-learn' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools installs the NLP package set" +else + _t_fail "install_python_nlp_tools should install scikit-learn and friends" +fi + +if grep -q 'ran:.*download_nltk_data.py' "${DEV}/scripts_run"; then + _t_pass "install_python_nlp_tools runs download_nltk_data.py" +else + _t_fail "install_python_nlp_tools should run download_nltk_data.py" +fi + +if grep -q '^python3 -m spacy download' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools downloads the spaCy model" +else + _t_fail "install_python_nlp_tools should download the spaCy model" +fi + +if [[ -x "${INSTALL_DIR}/check_plagiarism.py" ]]; then + _t_pass "install_python_nlp_tools installs check_plagiarism.py, executable" +else + _t_fail "install_python_nlp_tools should install an executable check_plagiarism.py" +fi + +if [[ -f "${INSTALL_DIR}/plagiarism_similarity.py" ]]; then + _t_pass "install_python_nlp_tools installs plagiarism_similarity.py alongside it" +else + _t_fail "install_python_nlp_tools should install plagiarism_similarity.py -- \ +check_plagiarism.py imports it, so a missing copy breaks at runtime, not install time" +fi + +if [[ -x "${HOME}/.local/bin/plagcheck" ]]; then + _t_pass "install_python_nlp_tools creates an executable plagcheck wrapper" +else + _t_fail "install_python_nlp_tools should create an executable plagcheck wrapper" +fi + +if grep -q "${VENV_DIR}/bin/activate" "${HOME}/.local/bin/plagcheck"; then + _t_pass "plagcheck wrapper activates the venv it was installed against" +else + _t_fail "plagcheck wrapper should reference ${VENV_DIR}/bin/activate" +fi + +if grep -q "${INSTALL_DIR}/check_plagiarism.py" "${HOME}/.local/bin/plagcheck"; then + _t_pass "plagcheck wrapper points at the installed checker script" +else + _t_fail "plagcheck wrapper should point at ${INSTALL_DIR}/check_plagiarism.py" +fi + +# --- venv already exists: warn, don't recreate -------------------------- + +reset_state +mkdir -p "${VENV_DIR}/bin" +cat >"${VENV_DIR}/bin/activate" <<'ACTEOF' +deactivate() { :; } +pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; } +ACTEOF +: >"${DEV}/calls" +install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1 +if grep -q '^python3 -m venv' "${DEV}/calls"; then + _t_fail "install_python_nlp_tools should not recreate an existing venv" +else + _t_pass "install_python_nlp_tools skips venv creation when one already exists" +fi + +# --- python3 missing: fatal, not silently skipped ------------------------ + +reset_state +# A bare `rm -f "${FAKE_BIN}/python3"` does not hide python3 from +# `command -v`: pyenv's shim is still reachable further down the real PATH. +# Instead point PATH at a directory that shadows python3 with nothing before +# falling through to the rest of the fakes, so `command -v python3` fails. +NO_PYTHON_BIN="${TEST_TMPDIR}/no_python_bin" +mkdir -p "${NO_PYTHON_BIN}" +for tool in git java wine cat chmod mkdir; do + [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_PYTHON_BIN}/${tool}" +done +# install_python_nlp_tools calls `exit` (not `return`) on a missing python3, +# by design -- a partial install with no interpreter must abort the whole +# installer, not just this stage. Run it in a subshell so that exit only +# ends the subshell, not this test script. +if (PATH="${NO_PYTHON_BIN}" install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1); then + _t_fail "install_python_nlp_tools should exit non-zero without python3" +else + _t_pass "install_python_nlp_tools exits non-zero when python3 is missing" +fi +if grep -q "Python 3 is required" "${DEV}/errors"; then + _t_pass "install_python_nlp_tools reports the missing-python3 error" +else + _t_fail "install_python_nlp_tools should report the missing-python3 error" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh new file mode 100755 index 0000000..5abaa0e --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Unit tests for lib/plagiarism_sherlock.sh against the fake git in +# plagiarism_harness.sh. No real network is needed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +# --- upstream clone succeeds --------------------------------------------- + +install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if grep -q '^git clone' "${DEV}/calls"; then + _t_pass "install_sherlock clones the upstream sherlock-py tool" +else + _t_fail "install_sherlock should attempt a git clone" +fi + +if [[ -d "${INSTALL_DIR}/sherlock" ]]; then + _t_pass "install_sherlock creates the sherlock directory on a successful clone" +else + _t_fail "install_sherlock should create ${INSTALL_DIR}/sherlock" +fi + +if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then + _t_fail "a successful clone should not also install the bundled fallback sherlock.py" +else + _t_pass "a successful clone does not install the bundled fallback sherlock.py" +fi + +# --- already installed: no-op, warns ------------------------------------- + +: >"${DEV}/calls" +install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if grep -q '^git clone' "${DEV}/calls"; then + _t_fail "install_sherlock should not re-clone when already installed" +else + _t_pass "install_sherlock skips reinstalling when the directory already exists" +fi + +if grep -q "already installed" "${DEV}/warnings"; then + _t_pass "install_sherlock warns that sherlock is already installed" +else + _t_fail "install_sherlock should warn that sherlock is already installed" +fi + +# --- clone fails: falls back to the bundled sherlock.py ------------------- + +reset_state +touch "${DEV}/fail_clone" +install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then + _t_pass "install_sherlock falls back to the bundled sherlock.py when the clone fails" +else + _t_fail "install_sherlock should fall back to the bundled sherlock.py when the clone fails" +fi + +if [[ -x "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then + _t_pass "the bundled fallback sherlock.py is installed executable" +else + _t_fail "the bundled fallback sherlock.py should be installed executable" +fi + +if grep -q "Could not clone sherlock-py" "${DEV}/warnings"; then + _t_pass "install_sherlock warns when the clone fails, before falling back" +else + _t_fail "install_sherlock should warn when the clone fails" +fi + +# The installed fallback must be byte-identical to the source file: it is the +# one artifact the git-unavailable path can serve, so a stale copy would ship +# broken plagiarism detection silently. +if diff -q "${UTILS_DIR}/sherlock.py" "${INSTALL_DIR}/sherlock/sherlock.py" >/dev/null; then + _t_pass "the installed fallback sherlock.py matches the source file byte-for-byte" +else + _t_fail "the installed fallback sherlock.py should match ${UTILS_DIR}/sherlock.py" +fi + +# --- git not available: warns, does not attempt to install ---------------- + +reset_state +NO_GIT_BIN="${TEST_TMPDIR}/no_git_bin" +mkdir -p "${NO_GIT_BIN}" +for tool in python3 java wine cat chmod mkdir; do + [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_GIT_BIN}/${tool}" +done +PATH="${NO_GIT_BIN}" install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if [[ -d "${INSTALL_DIR}/sherlock" ]]; then + _t_fail "install_sherlock should not create the sherlock directory without git" +else + _t_pass "install_sherlock skips installation entirely when git is unavailable" +fi + +if grep -q "Git not available" "${DEV}/warnings"; then + _t_pass "install_sherlock warns when git is unavailable" +else + _t_fail "install_sherlock should warn when git is unavailable" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py b/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py new file mode 100755 index 0000000..b11bee8 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Document reading and TF-IDF similarity scoring for check_plagiarism.py. + +Split out of `check_plagiarism.py` to keep it under the 250-line cap: this +module holds the comparison engine (file reading, text preprocessing, and the +TF-IDF/cosine-similarity computations), `check_plagiarism.py` keeps the CLI +and reporting. + +Its third-party imports (numpy, sklearn, nltk) are the plagiarism toolchain's +dependencies, not this repo's, so they are absent from `meta/requirements.txt` +on purpose. Importing them dynamically (rather than with a static `import +nltk`) keeps this module importable -- and lintable -- in an environment +that doesn't have them installed, matching `download_nltk_data.py`'s +extraction. +""" + +from __future__ import annotations + +import importlib +from pathlib import Path +import sys + +#: Cosine similarity above this, for a single sentence pair, is worth reporting +#: as a suspicious passage in --detailed mode. +PASSAGE_SIMILARITY_THRESHOLD = 0.5 + +nltk = importlib.import_module("nltk") +stopwords = importlib.import_module("nltk.corpus").stopwords +nltk_tokenize = importlib.import_module("nltk.tokenize") +sent_tokenize = nltk_tokenize.sent_tokenize +word_tokenize = nltk_tokenize.word_tokenize +TfidfVectorizer = importlib.import_module( + "sklearn.feature_extraction.text" +).TfidfVectorizer +cosine_similarity = importlib.import_module( + "sklearn.metrics.pairwise" +).cosine_similarity + +try: + stopwords.words("english") +except LookupError: + nltk.download("stopwords", quiet=True) + nltk.download("punkt", quiet=True) + + +def _out(line: str = "") -> None: + sys.stdout.write(line + "\n") + + +def _read_pdf(filepath: str) -> str: + try: + pypdf2 = importlib.import_module("PyPDF2") + except ImportError: + _out("Warning: PyPDF2 not installed, cannot read PDF files") + return "" + reader = pypdf2.PdfReader(filepath) + return " ".join(page.extract_text() or "" for page in reader.pages) + + +def _read_docx(filepath: str) -> str: + try: + docx = importlib.import_module("docx") + except ImportError: + _out("Warning: python-docx not installed, cannot read DOCX files") + return "" + doc = docx.Document(filepath) + return " ".join(para.text for para in doc.paragraphs) + + +def read_file(filepath: str) -> str: + """Read text from a .pdf, .docx, or plain-text file.""" + path = Path(filepath) + suffix = path.suffix.lower() + + if suffix == ".pdf": + return _read_pdf(filepath) + if suffix == ".docx": + return _read_docx(filepath) + with path.open(encoding="utf-8", errors="ignore") as f: + return f.read() + + +def preprocess_text(text: str) -> str: + """Lowercase, tokenize, and strip stopwords/non-alphanumerics.""" + text = text.lower() + try: + stop_words = set(stopwords.words("english")) + words = word_tokenize(text) + words = [w for w in words if w.isalnum() and w not in stop_words] + return " ".join(words) + except LookupError: + return " ".join(text.split()) + + +def compute_similarity_matrix(documents: list[str]) -> list[list[float]]: + """Compute the TF-IDF cosine similarity matrix for a set of documents. + + Returns a plain nested list, not the numpy array `cosine_similarity` + produces: numpy is a plagiarism-toolchain dependency the installer + pip-installs at runtime, deliberately absent from `meta/requirements.txt`, + so mypy cannot follow it into this repo's type-checked surface + (`no-any-unimported`). Converting at this boundary keeps every other + signature in the module concretely typed. + """ + vectorizer = TfidfVectorizer(ngram_range=(1, 3), min_df=1, max_df=0.95) + tfidf_matrix = vectorizer.fit_transform(documents) + matrix: list[list[float]] = cosine_similarity(tfidf_matrix).tolist() + return matrix + + +def find_similar_passages( + text1: str, text2: str, min_words: int = 5 +) -> list[tuple[str, str, float]]: + """Find similar sentence-level passages between two texts.""" + sentences1 = [s for s in sent_tokenize(text1) if len(s.split()) >= min_words] + sentences2 = [s for s in sent_tokenize(text2) if len(s.split()) >= min_words] + + if not sentences1 or not sentences2: + return [] + + all_sentences = sentences1 + sentences2 + preprocessed = [preprocess_text(s) for s in all_sentences] + + try: + vectorizer = TfidfVectorizer(ngram_range=(1, 2)) + tfidf_matrix = vectorizer.fit_transform(preprocessed) + except ValueError: + return [] + + n1 = len(sentences1) + similarities = [] + + for i, s1 in enumerate(sentences1): + for j, s2 in enumerate(sentences2): + sim = cosine_similarity( + tfidf_matrix[i : i + 1], tfidf_matrix[n1 + j : n1 + j + 1] + )[0][0] + if sim > PASSAGE_SIMILARITY_THRESHOLD: + similarities.append((s1, s2, sim)) + + return sorted(similarities, key=lambda x: x[2], reverse=True) diff --git a/linux_configuration/scripts/single_use/utils/sherlock.py b/linux_configuration/scripts/single_use/utils/sherlock.py new file mode 100755 index 0000000..0e9460d --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/sherlock.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Sherlock - simple text plagiarism detector using n-gram fingerprinting. + +Extracted from `install_plagiarism_tools.sh`'s fallback path, which used to +write this out via a `cat <<'SHERLOCKEOF'` heredoc when cloning the real +sherlock-py failed. Inline heredoc Python is invisible to ruff, mypy, pylint, +bandit and pytest, so it lives in its own module instead, matching +`download_nltk_data.py`'s extraction. + +The installer copies this file into `$SHERLOCK_DIR/sherlock.py` only when +`git clone` of the upstream tool fails. +""" + +from __future__ import annotations + +import argparse +import hashlib +from itertools import combinations +from pathlib import Path +import sys + +#: Minimum number of input files for a pairwise comparison to make sense. +MIN_FILES = 2 + + +def _out(line: str = "") -> None: + sys.stdout.write(line + "\n") + + +def tokenize(text: str) -> list[str]: + """Split text into lowercase alphanumeric words.""" + return [w.lower() for w in text.split() if w.isalnum()] + + +def get_ngrams(tokens: list[str], n: int = 3) -> list[tuple[str, ...]]: + """Generate n-grams from a token list.""" + return [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)] + + +def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set[str]: + """Create a document fingerprint from sampled n-gram hashes.""" + tokens = tokenize(text) + ngrams = get_ngrams(tokens, ngram_size) + + fingerprints = set() + for i, ng in enumerate(ngrams): + if i % sample_rate == 0: + h = hashlib.md5("".join(ng).encode(), usedforsecurity=False).hexdigest()[:8] + fingerprints.add(h) + + return fingerprints + + +def compare_documents(fp1: set[str], fp2: set[str]) -> float: + """Jaccard similarity between two fingerprints.""" + if not fp1 or not fp2: + return 0.0 + intersection = len(fp1 & fp2) + union = len(fp1 | fp2) + return intersection / union if union > 0 else 0.0 + + +def read_document(filepath: str) -> str: + """Read a document's raw text content.""" + with Path(filepath).open(encoding="utf-8", errors="ignore") as f: + return f.read() + + +def main() -> int: + """Fingerprint every input file and report pairwise similarity.""" + parser = argparse.ArgumentParser(description="Sherlock - Text Plagiarism Detector") + parser.add_argument("files", nargs="+", help="Files to compare") + parser.add_argument( + "--ngram", "-n", type=int, default=3, help="N-gram size (default: 3)" + ) + parser.add_argument( + "--threshold", "-t", type=float, default=0.1, help="Similarity threshold" + ) + + args = parser.parse_args() + + if len(args.files) < MIN_FILES: + _out("Need at least 2 files to compare") + return 1 + + docs = {} + for f in args.files: + if Path(f).exists(): + text = read_document(f) + docs[f] = fingerprint(text, args.ngram) + + _out("\nSherlock Plagiarism Analysis") + _out("=" * 50) + + files = list(docs.keys()) + for file1, file2 in combinations(files, 2): + sim = compare_documents(docs[file1], docs[file2]) + name1 = Path(file1).name + name2 = Path(file2).name + flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else "" + _out(f"{name1} <-> {name2}: {sim:.1%}{flag}") + + _out("=" * 50) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())