From 4e58ff837ea9caa5ce100439420db8dd92728506 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 6 Feb 2026 21:43:26 +0100 Subject: [PATCH 1/5] Add 'linux_configuration/' from commit '0762e3d07b90bac9256eb272de10bf9f42878094' git-subtree-dir: linux_configuration git-subtree-mainline: 11427631cd6779dffaed92b26a3e7b7fdf72b9d7 git-subtree-split: 0762e3d07b90bac9256eb272de10bf9f42878094 --- .../scripts/utils/install_plagiarism_tools.sh | 532 ++++++++++++++++++ 1 file changed, 532 insertions(+) create mode 100755 linux_configuration/scripts/utils/install_plagiarism_tools.sh diff --git a/linux_configuration/scripts/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/utils/install_plagiarism_tools.sh new file mode 100755 index 0000000..1de0b64 --- /dev/null +++ b/linux_configuration/scripts/utils/install_plagiarism_tools.sh @@ -0,0 +1,532 @@ +#!/usr/bin/env bash +# Install Free & Open Source Plagiarism Detection Tools for Text +# Suitable for academic work (theses, papers, etc.) +# +# Tools installed: +# 1. Python NLP-based similarity detection (sklearn, NLTK, spaCy) +# 2. Sherlock text plagiarism detector +# 3. Ferret (Java-based, if Java available) +# 4. Optional: WCopyfind via Wine (Windows tool) + +set -euo pipefail + +INSTALL_DIR="${HOME}/.local/share/plagiarism-tools" +VENV_DIR="${HOME}/.local/share/plagiarism-venv" + +echo "==============================================" +echo " Open Source Plagiarism Detection Installer" +echo " For Academic Text (Theses, Papers, etc.)" +echo "==============================================" +echo "" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +success() { echo -e "${GREEN}✓ $1${NC}"; } +warn() { echo -e "${YELLOW}⚠ $1${NC}"; } +error() { echo -e "${RED}✗ $1${NC}"; } + +# Create installation directory +mkdir -p "$INSTALL_DIR" + +# ------------------------------------------------------------------------------ +# 1. Python-based NLP Plagiarism Detection Environment +# ------------------------------------------------------------------------------ +echo "" +echo "=== 1. Installing Python NLP-based Plagiarism Tools ===" + +# Check for Python 3 +if ! command -v python3 &> /dev/null; then + error "Python 3 is required but not installed." + exit 1 +fi + +# Create virtual environment +if [ ! -d "$VENV_DIR" ]; then + echo "Creating Python virtual environment..." + python3 -m venv "$VENV_DIR" + success "Virtual environment created at $VENV_DIR" +else + warn "Virtual environment already exists at $VENV_DIR" +fi + +# Activate and install packages +source "$VENV_DIR/bin/activate" + +echo "Installing Python packages for text similarity detection..." +pip install --upgrade pip + +pip install --progress-bar on \ + scikit-learn \ + nltk \ + spacy \ + gensim \ + numpy \ + pandas \ + python-docx \ + PyPDF2 \ + beautifulsoup4 \ + lxml \ + textdistance \ + fuzzywuzzy \ + python-Levenshtein + +success "Python NLP packages installed" + +# Download NLTK data +echo "Downloading NLTK data (stopwords, punkt tokenizer)..." +python3 -c " +import nltk +nltk.download('punkt') +nltk.download('stopwords') +nltk.download('punkt_tab') +nltk.download('averaged_perceptron_tagger') +nltk.download('wordnet') +" +success "NLTK data downloaded" + +# Download spaCy English model (small) +echo "Downloading spaCy English model..." +python3 -m spacy download en_core_web_sm 2> /dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm" +success "spaCy model installed" + +# Create a simple plagiarism checker script +cat > "$INSTALL_DIR/check_plagiarism.py" << 'PYEOF' +#!/usr/bin/env python3 +""" +Simple Text Plagiarism Checker +Compares documents using multiple similarity algorithms. + +Usage: + python check_plagiarism.py file1.txt file2.txt [file3.txt ...] + python check_plagiarism.py --dir /path/to/documents/ +""" + +import argparse +import os +import sys +from pathlib import Path +from typing import List, Tuple + +import numpy as np +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.metrics.pairwise import cosine_similarity +import nltk +from nltk.corpus import stopwords +from nltk.tokenize import word_tokenize, sent_tokenize + +# Ensure NLTK data is available +try: + stopwords.words('english') +except LookupError: + nltk.download('stopwords', quiet=True) + nltk.download('punkt', quiet=True) + + +def read_file(filepath: str) -> str: + """Read text from various file formats.""" + path = Path(filepath) + suffix = path.suffix.lower() + + if suffix == '.pdf': + try: + from PyPDF2 import PdfReader + reader = PdfReader(filepath) + return ' '.join(page.extract_text() or '' for page in reader.pages) + except ImportError: + print("Warning: PyPDF2 not installed, cannot read PDF files") + return "" + elif suffix == '.docx': + try: + from docx import Document + doc = Document(filepath) + return ' '.join(para.text for para in doc.paragraphs) + except ImportError: + print("Warning: python-docx not installed, cannot read DOCX files") + return "" + else: + # Assume plain text + with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: + return f.read() + + +def preprocess_text(text: str) -> str: + """Clean and preprocess text for comparison.""" + # Lowercase + text = text.lower() + # Tokenize and remove stopwords + try: + stop_words = set(stopwords.words('english')) + words = word_tokenize(text) + words = [w for w in words if w.isalnum() and w not in stop_words] + return ' '.join(words) + except Exception: + # Fallback: simple preprocessing + return ' '.join(text.split()) + + +def compute_similarity_matrix(documents: List[str]) -> np.ndarray: + """Compute TF-IDF cosine similarity matrix.""" + vectorizer = TfidfVectorizer( + ngram_range=(1, 3), # Use unigrams, bigrams, trigrams + min_df=1, + max_df=0.95 + ) + tfidf_matrix = vectorizer.fit_transform(documents) + return cosine_similarity(tfidf_matrix) + + +def find_similar_passages(text1: str, text2: str, min_words: int = 5) -> List[Tuple[str, str, float]]: + """Find similar sentence-level passages between two texts.""" + sentences1 = sent_tokenize(text1) + sentences2 = sent_tokenize(text2) + + if not sentences1 or not sentences2: + return [] + + # Filter short sentences + sentences1 = [s for s in sentences1 if len(s.split()) >= min_words] + sentences2 = [s for s in sentences2 if len(s.split()) >= min_words] + + if not sentences1 or not sentences2: + return [] + + all_sentences = sentences1 + sentences2 + preprocessed = [preprocess_text(s) for s in all_sentences] + + try: + vectorizer = TfidfVectorizer(ngram_range=(1, 2)) + tfidf_matrix = vectorizer.fit_transform(preprocessed) + except ValueError: + return [] + + n1 = len(sentences1) + similarities = [] + + for i, s1 in enumerate(sentences1): + for j, s2 in enumerate(sentences2): + sim = cosine_similarity( + tfidf_matrix[i:i+1], + tfidf_matrix[n1+j:n1+j+1] + )[0][0] + if sim > 0.5: # Threshold for suspicious similarity + similarities.append((s1, s2, sim)) + + return sorted(similarities, key=lambda x: x[2], reverse=True) + + +def main(): + parser = argparse.ArgumentParser( + description='Text Plagiarism Checker - Compare documents for similarity' + ) + parser.add_argument('files', nargs='*', help='Files to compare') + parser.add_argument('--dir', '-d', help='Directory containing documents to compare') + parser.add_argument('--threshold', '-t', type=float, default=0.3, + help='Similarity threshold for flagging (0-1, default: 0.3)') + parser.add_argument('--detailed', '-v', action='store_true', + help='Show detailed similar passages') + + args = parser.parse_args() + + # Collect files + files = [] + if args.files: + files.extend(args.files) + if args.dir: + dir_path = Path(args.dir) + for ext in ['*.txt', '*.pdf', '*.docx', '*.md', '*.tex']: + files.extend(str(f) for f in dir_path.glob(ext)) + + if len(files) < 2: + print("Error: Need at least 2 files to compare") + parser.print_help() + sys.exit(1) + + print(f"\n{'='*60}") + print(f" Plagiarism Check - Analyzing {len(files)} documents") + print(f"{'='*60}\n") + + # Read and preprocess documents + documents = [] + filenames = [] + for f in files: + if os.path.exists(f): + text = read_file(f) + if text.strip(): + documents.append(preprocess_text(text)) + filenames.append(os.path.basename(f)) + else: + print(f"Warning: {f} is empty or unreadable") + else: + print(f"Warning: {f} does not exist") + + if len(documents) < 2: + print("Error: Not enough valid documents to compare") + sys.exit(1) + + # Compute similarity + print("Computing document similarities...\n") + sim_matrix = compute_similarity_matrix(documents) + + # Report results + print(f"{'Document Pair':<50} {'Similarity':>12}") + print("-" * 62) + + suspicious_pairs = [] + for i in range(len(documents)): + for j in range(i + 1, len(documents)): + similarity = sim_matrix[i][j] + pair_name = f"{filenames[i]} <-> {filenames[j]}" + + if similarity >= args.threshold: + suspicious_pairs.append((i, j, similarity, pair_name)) + print(f"{pair_name:<50} {similarity:>10.1%} ⚠️") + else: + print(f"{pair_name:<50} {similarity:>10.1%}") + + print("-" * 62) + + # Summary + if suspicious_pairs: + print(f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed {args.threshold:.0%} similarity threshold\n") + + if args.detailed: + print("\n" + "="*60) + print(" Detailed Similar Passages") + print("="*60) + + for i, j, sim, pair_name in suspicious_pairs[:3]: # Limit to top 3 + print(f"\n{pair_name} ({sim:.1%} similar):") + print("-" * 40) + + raw_docs = [read_file(files[i]), read_file(files[j])] + passages = find_similar_passages(raw_docs[0], raw_docs[1]) + + for s1, s2, psim in passages[:5]: # Top 5 passages + print(f"\n[{psim:.0%}] Document 1: \"{s1[:100]}...\"") + print(f" Document 2: \"{s2[:100]}...\"") + else: + print(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold") + + print("\n" + "="*60) + print(" Analysis complete") + print("="*60 + "\n") + + +if __name__ == '__main__': + main() +PYEOF + +chmod +x "$INSTALL_DIR/check_plagiarism.py" +success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py" + +# Create convenience wrapper +mkdir -p "$HOME/.local/bin" +cat > "$HOME/.local/bin/plagcheck" << WRAPEOF +#!/usr/bin/env bash +# Wrapper for plagiarism checker +source "$VENV_DIR/bin/activate" +python "$INSTALL_DIR/check_plagiarism.py" "\$@" +WRAPEOF +chmod +x "$HOME/.local/bin/plagcheck" +success "Created 'plagcheck' command in ~/.local/bin/" + +deactivate + +# ------------------------------------------------------------------------------ +# 2. Sherlock for Text (Clone from GitHub) +# ------------------------------------------------------------------------------ +echo "" +echo "=== 2. Installing Sherlock Text Plagiarism Detector ===" + +SHERLOCK_DIR="$INSTALL_DIR/sherlock" +if [ ! -d "$SHERLOCK_DIR" ]; then + # There are several Sherlock implementations; using a popular Python one + if command -v git &> /dev/null; then + # Clone a text-based similarity tool + git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$SHERLOCK_DIR" 2> /dev/null || { + warn "Could not clone sherlock-py, trying alternative..." + # Alternative: Create a simple n-gram based sherlock + mkdir -p "$SHERLOCK_DIR" + cat > "$SHERLOCK_DIR/sherlock.py" << 'SHERLOCKEOF' +#!/usr/bin/env python3 +""" +Sherlock - Simple text plagiarism detector using n-gram fingerprinting. +Based on the original Sherlock algorithm. +""" + +import argparse +import hashlib +import os +import sys +from collections import defaultdict +from pathlib import Path + + +def tokenize(text: str) -> list: + """Simple word tokenization.""" + return [w.lower() for w in text.split() if w.isalnum()] + + +def get_ngrams(tokens: list, n: int = 3) -> list: + """Generate n-grams from token list.""" + return [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)] + + +def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set: + """Create document fingerprint using sampled n-gram hashes.""" + tokens = tokenize(text) + ngrams = get_ngrams(tokens, ngram_size) + + fingerprints = set() + for i, ng in enumerate(ngrams): + if i % sample_rate == 0: # Sample every nth n-gram + h = hashlib.md5(''.join(ng).encode()).hexdigest()[:8] + fingerprints.add(h) + + return fingerprints + + +def compare_documents(fp1: set, fp2: set) -> float: + """Jaccard similarity between fingerprints.""" + if not fp1 or not fp2: + return 0.0 + intersection = len(fp1 & fp2) + union = len(fp1 | fp2) + return intersection / union if union > 0 else 0.0 + + +def read_document(filepath: str) -> str: + """Read document content.""" + with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: + return f.read() + + +def main(): + parser = argparse.ArgumentParser(description='Sherlock - Text Plagiarism Detector') + parser.add_argument('files', nargs='+', help='Files to compare') + parser.add_argument('--ngram', '-n', type=int, default=3, help='N-gram size (default: 3)') + parser.add_argument('--threshold', '-t', type=float, default=0.1, help='Similarity threshold') + + args = parser.parse_args() + + if len(args.files) < 2: + print("Need at least 2 files to compare") + sys.exit(1) + + # Read and fingerprint documents + docs = {} + for f in args.files: + if os.path.exists(f): + text = read_document(f) + docs[f] = fingerprint(text, args.ngram) + + print(f"\nSherlock Plagiarism Analysis") + print("=" * 50) + + # Compare all pairs + files = list(docs.keys()) + for i in range(len(files)): + for j in range(i + 1, len(files)): + sim = compare_documents(docs[files[i]], docs[files[j]]) + name1 = os.path.basename(files[i]) + name2 = os.path.basename(files[j]) + flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else "" + print(f"{name1} <-> {name2}: {sim:.1%}{flag}") + + print("=" * 50) + + +if __name__ == '__main__': + main() +SHERLOCKEOF + chmod +x "$SHERLOCK_DIR/sherlock.py" + } + success "Sherlock installed at $SHERLOCK_DIR" + else + warn "Git not available, skipping Sherlock installation" + fi +else + warn "Sherlock already installed at $SHERLOCK_DIR" +fi + +# ------------------------------------------------------------------------------ +# 3. Ferret (Java-based) - Optional +# ------------------------------------------------------------------------------ +echo "" +echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ===" + +if command -v java &> /dev/null; then + FERRET_DIR="$INSTALL_DIR/ferret" + if [ ! -d "$FERRET_DIR" ]; then + mkdir -p "$FERRET_DIR" + echo "Ferret is a Java-based tool from University of Hertfordshire." + echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/" + echo "Place JAR file in: $FERRET_DIR" + warn "Ferret requires manual download (academic license)" + fi +else + warn "Java not installed, skipping Ferret" +fi + +# ------------------------------------------------------------------------------ +# 4. WCopyfind via Wine (Optional) +# ------------------------------------------------------------------------------ +echo "" +echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ===" + +if command -v wine &> /dev/null; then + echo "Wine is available. WCopyfind can be run via Wine." + echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + echo "Run with: wine /path/to/WCopyfind.exe" + warn "WCopyfind requires manual download" +else + echo "Wine not installed. To use WCopyfind:" + echo " 1. Install wine: sudo apt install wine (or equivalent)" + echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + warn "WCopyfind skipped (Wine not available)" +fi + +# ------------------------------------------------------------------------------ +# Summary +# ------------------------------------------------------------------------------ +echo "" +echo "==============================================" +echo " Installation Complete!" +echo "==============================================" +echo "" +echo "Installed tools:" +echo "" +echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)" +echo " Usage: plagcheck file1.txt file2.txt" +echo " plagcheck --dir /path/to/documents/ --detailed" +echo " Location: $INSTALL_DIR/check_plagiarism.py" +echo "" +echo "2. Sherlock (n-gram fingerprinting)" +echo " Location: $SHERLOCK_DIR/sherlock.py" +echo "" +echo "3. Python virtual environment with NLP libraries:" +echo " - scikit-learn (TF-IDF, cosine similarity)" +echo " - NLTK (tokenization, stopwords)" +echo " - spaCy (NLP processing)" +echo " - gensim (document similarity)" +echo " - textdistance, fuzzywuzzy (string matching)" +echo " Activate with: source $VENV_DIR/bin/activate" +echo "" +echo "Quick Start:" +echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed" +echo " plagcheck --dir ./student_papers/ --threshold 0.4" +echo "" +echo "Note: Ensure ~/.local/bin is in your PATH:" +echo ' export PATH="$HOME/.local/bin:$PATH"' +echo "" +echo "==============================================" + +# Add to PATH reminder +if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:" + echo ' export PATH="$HOME/.local/bin:$PATH"' +fi From 7631cb31bc9b30258243e686e87bc6c61e0ea699 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 15 May 2026 00:32:35 +0200 Subject: [PATCH 2/5] refactor(linux_configuration/scripts): split all scripts into single_use/ and periodic_background/ - Move all linux_configuration scripts into two semantic categories: - single_use/: scripts run once manually (fresh install, fixes, setup) - periodic_background/: scripts run by systemd timers or daemons - Preserve existing subdirectory structure within each category - Fix lib/common.sh source paths for new directory depths - Fix CONFIG_DIR depth in setup_periodic_system.sh and check_and_enable_services.sh - Update all references in tests, fresh-install/main.sh, nix modules, and docs - Fix check_polling_antipatterns.sh false positives (||, regex |, case patterns, jq strings) - Fix pre-existing mypy exclusion path and type annotations for moved tools/ directory - Rewrite check_polling_antipatterns.sh using awk (no bash regex loops); add require_serial: true --- .../scripts/{ => single_use}/utils/install_plagiarism_tools.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename linux_configuration/scripts/{ => single_use}/utils/install_plagiarism_tools.sh (100%) diff --git a/linux_configuration/scripts/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh similarity index 100% rename from linux_configuration/scripts/utils/install_plagiarism_tools.sh rename to linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh From a778b66b15f9156c57d3d7a7167d5b18d9fa7175 Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Mon, 3 Aug 2026 18:34:49 +0200 Subject: [PATCH 3/5] Fail the build on any shellcheck finding; fix the 222 the old gate hid The shellcheck hook ran with --severity=warning, so info- and style-level findings never failed a commit. 222 had accumulated across 94 of 244 shell files. They were not all style noise -- six were bugs: - Five scripts in single_use/misc/testsAndMisc-bash/ sourced $SCRIPT_DIR/../../lib/common.sh, one directory too shallow. Nothing exists at that path, so all five failed at runtime. - setup_personal_website.sh and setup_kcd2_dice_solver.sh called usage() (which ends in `exit 0`) and then `exit 1` for an unknown command. The exit 1 was unreachable, so a typo'd subcommand reported SUCCESS. - Six i3blocks scripts had a `# shellcheck source=` directive missing the scripts/periodic_background/ path segment. The stale path had silently disabled source-following for all of them. - diagnose_arch_performance.sh word-split a newline-joined file list; when the find came back empty, awk got no file operands, fell through to reading stdin, and the diagnostic hung. - lib/common.sh's handle_arg_help_or_unknown ended in an unreachable `return 0` while documenting "Returns: 0 if handled" -- a contract no code path could satisfy. - deploy.sh's SCRIPT_DIR is reassigned by config.sh, which it sources at top level, and config.sh can repoint it at the on-device path /data/local/tmp/focus_mode. Renamed to DEPLOY_DIR. The SC1091 bulk (81 findings) was resolved rather than silenced: .shellcheckrc sets external-sources + source-path=SCRIPTDIR so sourced files are actually followed and analysed. That is what surfaced the i3blocks and deploy.sh bugs. 17 `# shellcheck disable=` directives remain, each with a written reason, and only where the finding is provably wrong: injection-test payloads that must not expand, literal source text being grepped for, POSIX sh scripts on the phone where arrays do not exist, and source paths that only exist at runtime. Everything else was fixed in code. Verified in a CLEAN worktree, not just my own tree. config.sh sources the gitignored config_secrets.sh, which exists locally but not in a fresh checkout; a working-tree-only check passed while ci_mirror.sh -- which lints HEAD in a clean worktree, exactly as CI does -- failed on it. The gate is verified to bite: a file whose only finding is SC2016 (info) fails the hook, while the old --severity=warning gate exits 0 on it. third_party/ is excluded -- vendored upstream, matching the prettier hook. Not fixed, reported: check_polling_antipatterns.sh carries a pre-existing SC2034 disable that hides a real bug -- its `patterns` array is never used, so the checker detects nothing. Confirmed unchanged by this commit. Co-Authored-By: Claude Opus 5 (1M context) --- .../utils/install_plagiarism_tools.sh | 127 ++++++++++-------- 1 file changed, 68 insertions(+), 59 deletions(-) diff --git a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh index 1de0b64..ed8b457 100755 --- a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh +++ b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh @@ -39,40 +39,43 @@ echo "" echo "=== 1. Installing Python NLP-based Plagiarism Tools ===" # Check for Python 3 -if ! command -v python3 &> /dev/null; then - error "Python 3 is required but not installed." - exit 1 +if ! command -v python3 &>/dev/null; then + error "Python 3 is required but not installed." + exit 1 fi # Create virtual environment if [ ! -d "$VENV_DIR" ]; then - echo "Creating Python virtual environment..." - python3 -m venv "$VENV_DIR" - success "Virtual environment created at $VENV_DIR" + echo "Creating Python virtual environment..." + python3 -m venv "$VENV_DIR" + success "Virtual environment created at $VENV_DIR" else - warn "Virtual environment already exists at $VENV_DIR" + warn "Virtual environment already exists at $VENV_DIR" fi # Activate and install packages +# Created at runtime by `python -m venv` above, so there is nothing on disk +# for the linter to follow at check time. +# shellcheck disable=SC1091 source "$VENV_DIR/bin/activate" echo "Installing Python packages for text similarity detection..." pip install --upgrade pip pip install --progress-bar on \ - scikit-learn \ - nltk \ - spacy \ - gensim \ - numpy \ - pandas \ - python-docx \ - PyPDF2 \ - beautifulsoup4 \ - lxml \ - textdistance \ - fuzzywuzzy \ - python-Levenshtein + scikit-learn \ + nltk \ + spacy \ + gensim \ + numpy \ + pandas \ + python-docx \ + PyPDF2 \ + beautifulsoup4 \ + lxml \ + textdistance \ + fuzzywuzzy \ + python-Levenshtein success "Python NLP packages installed" @@ -90,11 +93,11 @@ success "NLTK data downloaded" # Download spaCy English model (small) echo "Downloading spaCy English model..." -python3 -m spacy download en_core_web_sm 2> /dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm" +python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm" success "spaCy model installed" # Create a simple plagiarism checker script -cat > "$INSTALL_DIR/check_plagiarism.py" << 'PYEOF' +cat >"$INSTALL_DIR/check_plagiarism.py" <<'PYEOF' #!/usr/bin/env python3 """ Simple Text Plagiarism Checker @@ -325,7 +328,7 @@ success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py" # Create convenience wrapper mkdir -p "$HOME/.local/bin" -cat > "$HOME/.local/bin/plagcheck" << WRAPEOF +cat >"$HOME/.local/bin/plagcheck" < /dev/null; then - # Clone a text-based similarity tool - git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$SHERLOCK_DIR" 2> /dev/null || { - warn "Could not clone sherlock-py, trying alternative..." - # Alternative: Create a simple n-gram based sherlock - mkdir -p "$SHERLOCK_DIR" - cat > "$SHERLOCK_DIR/sherlock.py" << 'SHERLOCKEOF' + # There are several Sherlock implementations; using a popular Python one + if command -v git &>/dev/null; then + # Clone a text-based similarity tool + git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$SHERLOCK_DIR" 2>/dev/null || { + warn "Could not clone sherlock-py, trying alternative..." + # Alternative: Create a simple n-gram based sherlock + mkdir -p "$SHERLOCK_DIR" + cat >"$SHERLOCK_DIR/sherlock.py" <<'SHERLOCKEOF' #!/usr/bin/env python3 """ Sherlock - Simple text plagiarism detector using n-gram fingerprinting. @@ -443,14 +446,14 @@ def main(): if __name__ == '__main__': main() SHERLOCKEOF - chmod +x "$SHERLOCK_DIR/sherlock.py" - } - success "Sherlock installed at $SHERLOCK_DIR" - else - warn "Git not available, skipping Sherlock installation" - fi + chmod +x "$SHERLOCK_DIR/sherlock.py" + } + success "Sherlock installed at $SHERLOCK_DIR" + else + warn "Git not available, skipping Sherlock installation" + fi else - warn "Sherlock already installed at $SHERLOCK_DIR" + warn "Sherlock already installed at $SHERLOCK_DIR" fi # ------------------------------------------------------------------------------ @@ -459,17 +462,17 @@ fi echo "" echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ===" -if command -v java &> /dev/null; then - FERRET_DIR="$INSTALL_DIR/ferret" - if [ ! -d "$FERRET_DIR" ]; then - mkdir -p "$FERRET_DIR" - echo "Ferret is a Java-based tool from University of Hertfordshire." - echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/" - echo "Place JAR file in: $FERRET_DIR" - warn "Ferret requires manual download (academic license)" - fi +if command -v java &>/dev/null; then + FERRET_DIR="$INSTALL_DIR/ferret" + if [ ! -d "$FERRET_DIR" ]; then + mkdir -p "$FERRET_DIR" + echo "Ferret is a Java-based tool from University of Hertfordshire." + echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/" + echo "Place JAR file in: $FERRET_DIR" + warn "Ferret requires manual download (academic license)" + fi else - warn "Java not installed, skipping Ferret" + warn "Java not installed, skipping Ferret" fi # ------------------------------------------------------------------------------ @@ -478,16 +481,16 @@ fi echo "" echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ===" -if command -v wine &> /dev/null; then - echo "Wine is available. WCopyfind can be run via Wine." - echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" - echo "Run with: wine /path/to/WCopyfind.exe" - warn "WCopyfind requires manual download" +if command -v wine &>/dev/null; then + echo "Wine is available. WCopyfind can be run via Wine." + echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + echo "Run with: wine /path/to/WCopyfind.exe" + warn "WCopyfind requires manual download" else - echo "Wine not installed. To use WCopyfind:" - echo " 1. Install wine: sudo apt install wine (or equivalent)" - echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" - warn "WCopyfind skipped (Wine not available)" + echo "Wine not installed. To use WCopyfind:" + echo " 1. Install wine: sudo apt install wine (or equivalent)" + echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + warn "WCopyfind skipped (Wine not available)" fi # ------------------------------------------------------------------------------ @@ -521,12 +524,18 @@ echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed" echo " plagcheck --dir ./student_papers/ --threshold 0.4" echo "" echo "Note: Ensure ~/.local/bin is in your PATH:" +# Printed for the user to paste into their shell rc: $HOME and $PATH must +# stay literal rather than resolve to this machine's values. +# shellcheck disable=SC2016 echo ' export PATH="$HOME/.local/bin:$PATH"' echo "" echo "==============================================" # Add to PATH reminder if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then - warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:" - echo ' export PATH="$HOME/.local/bin:$PATH"' + warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:" + # Printed for the user to paste into their shell rc: $HOME and $PATH must + # stay literal rather than resolve to this machine's values. + # shellcheck disable=SC2016 + echo ' export PATH="$HOME/.local/bin:$PATH"' fi From 0186100a566ffa6ac8b0bf7ffc46031c99a1394a Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 16 Aug 2026 15:11:56 +0200 Subject: [PATCH 4/5] Gate the no-inline-Python rule instead of asking people to remember it CLAUDE.md's longest and most emphatic code rule -- never embed Python program logic in a shell script -- was enforced by nothing. A repo-wide scan found it had already been violated: install_plagiarism_tools.sh ran a 6-line NLTK download as an inline `python3 -c`, invisible to ruff, mypy, pylint, bandit and pytest, exactly the failure mode the rule exists to prevent. check_no_inline_python.sh flags heredocs and multi-line `python -c` while permitting the single-line availability probes the rule explicitly allows, so the ~250 shell files here produce zero false positives. Commented-out examples are skipped; otherwise the rule's own documentation would fail the hook. The extracted download_nltk_data.py also fixes a real defect: the inline block ignored nltk.download()'s return value, so a partial corpus download reported success. It now returns non-zero and the caller's `set -e` sees it. nltk is imported via importlib inside main(), not at module scope: it belongs to the plagiarism toolchain, not this repo, so it is absent from requirements.txt and unresolvable to pylint as a static import. No suppressions anywhere: T201 avoided with sys.stderr.write, INP001 with a shebang (matching sibling fast_count.py), E0401 with the dynamic import. pylint 10.00/10, ruff clean, mypy clean. Verified the gate fires rather than passing vacuously -- a staged canary with a real heredoc exits non-zero -- and that the extracted script runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY --- .../single_use/utils/download_nltk_data.py | 53 +++++++++++++++++++ .../utils/install_plagiarism_tools.sh | 9 +--- 2 files changed, 54 insertions(+), 8 deletions(-) create mode 100755 linux_configuration/scripts/single_use/utils/download_nltk_data.py diff --git a/linux_configuration/scripts/single_use/utils/download_nltk_data.py b/linux_configuration/scripts/single_use/utils/download_nltk_data.py new file mode 100755 index 0000000..0d3064a --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/download_nltk_data.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Download the NLTK corpora the plagiarism tools need. + +Extracted from `install_plagiarism_tools.sh`, which used to run this as an +inline `python3 -c` block. Inline Python is invisible to ruff, mypy, pylint, +bandit and pytest, so it lives in its own module instead. + +`nltk` is a dependency of the plagiarism toolchain, not of this repo, so it is +absent from `meta/requirements.txt` on purpose. The installer script pip-installs +it immediately before invoking this module, and probes for it first. +""" + +from __future__ import annotations + +import importlib +import sys + +#: Corpora required by the plagiarism checker. `punkt_tab` is the post-3.8.2 +#: replacement for `punkt`; both are listed so the script works on either. +CORPORA = ( + "punkt", + "stopwords", + "punkt_tab", + "averaged_perceptron_tagger", + "wordnet", +) + + +def main() -> int: + """Download every corpus, reporting any that fail. + + Returns 1 if any download failed, so the calling shell script's `set -e` + sees a real failure instead of a silent partial install. + """ + # Imported dynamically, not at module scope: nltk belongs to the plagiarism + # toolchain rather than this repo, so it is deliberately absent from + # meta/requirements.txt and unresolvable to static analysis. The installer + # pip-installs it immediately before invoking this module. + try: + nltk = importlib.import_module("nltk") + except ImportError: + sys.stderr.write("error: nltk is not installed\n") + return 1 + + failed = [corpus for corpus in CORPORA if not nltk.download(corpus)] + if failed: + sys.stderr.write(f"error: failed to download: {', '.join(failed)}\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh index ed8b457..3956090 100755 --- a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh +++ b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh @@ -81,14 +81,7 @@ success "Python NLP packages installed" # Download NLTK data echo "Downloading NLTK data (stopwords, punkt tokenizer)..." -python3 -c " -import nltk -nltk.download('punkt') -nltk.download('stopwords') -nltk.download('punkt_tab') -nltk.download('averaged_perceptron_tagger') -nltk.download('wordnet') -" +python3 "$(dirname "$0")/download_nltk_data.py" success "NLTK data downloaded" # Download spaCy English model (small) From e47c72329a1bfbd050af4f31ab3c129cbc7a42db Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Tue, 18 Aug 2026 20:41:18 +0200 Subject: [PATCH 5/5] install_plagiarism_tools.sh: split 534 -> 94 + 3 libs, with a real test harness Wraps the four install stages into named functions, one per lib/plagiarism_*.sh sibling (matching install_exercism.sh's convention), and extracts the two embedded Python heredocs into real, lint-clean check_plagiarism.py/sherlock.py files -- check_plagiarism.py needed a second split into plagiarism_similarity.py to also clear the 250-line cap. Every function reaches 100% shell coverage via a PATH-shim test harness (fake python3/pip/git/java/wine); the two Python CLIs were separately verified against a real venv with actual nltk/scikit-learn. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY --- .../single_use/utils/check_plagiarism.py | 165 +++++ .../utils/install_plagiarism_tools.sh | 570 ++---------------- .../utils/lib/plagiarism_optional.sh | 48 ++ .../single_use/utils/lib/plagiarism_python.sh | 80 +++ .../utils/lib/plagiarism_sherlock.sh | 40 ++ .../utils/lib/tests/plagiarism_harness.sh | 130 ++++ .../tests/test_install_plagiarism_tools.sh | 114 ++++ .../lib/tests/test_plagiarism_optional.sh | 103 ++++ .../utils/lib/tests/test_plagiarism_python.sh | 132 ++++ .../lib/tests/test_plagiarism_sherlock.sh | 106 ++++ .../single_use/utils/plagiarism_similarity.py | 141 +++++ .../scripts/single_use/utils/sherlock.py | 108 ++++ 12 files changed, 1232 insertions(+), 505 deletions(-) create mode 100755 linux_configuration/scripts/single_use/utils/check_plagiarism.py create mode 100755 linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/plagiarism_sherlock.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh create mode 100755 linux_configuration/scripts/single_use/utils/plagiarism_similarity.py create mode 100755 linux_configuration/scripts/single_use/utils/sherlock.py diff --git a/linux_configuration/scripts/single_use/utils/check_plagiarism.py b/linux_configuration/scripts/single_use/utils/check_plagiarism.py new file mode 100755 index 0000000..71bb08d --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/check_plagiarism.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Compare documents for similarity using TF-IDF cosine similarity. + +Extracted from `install_plagiarism_tools.sh`, which used to write this out via +a `cat <<'PYEOF'` heredoc. Inline heredoc Python is invisible to ruff, mypy, +pylint, bandit and pytest, so it lives in its own module instead, matching +`download_nltk_data.py`'s extraction. The comparison engine (file reading, +preprocessing, TF-IDF scoring) lives in `plagiarism_similarity.py`, split out +to keep both files under the 250-line cap. + +The installer copies this file into `$INSTALL_DIR/check_plagiarism.py` and +wires the `plagcheck` wrapper to run it inside the venv. + +Usage: + python check_plagiarism.py file1.txt file2.txt [file3.txt ...] + python check_plagiarism.py --dir /path/to/documents/ +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +from plagiarism_similarity import ( + compute_similarity_matrix, + find_similar_passages, + preprocess_text, + read_file, +) + +#: Minimum number of documents needed for a pairwise comparison to make sense. +MIN_DOCUMENTS = 2 + + +def _out(line: str = "") -> None: + sys.stdout.write(line + "\n") + + +def _collect_files(args: argparse.Namespace) -> list[str]: + files: list[str] = list(args.files) + if args.dir: + dir_path = Path(args.dir) + for ext in ("*.txt", "*.pdf", "*.docx", "*.md", "*.tex"): + files.extend(str(f) for f in dir_path.glob(ext)) + return files + + +def _read_documents(files: list[str]) -> tuple[list[str], list[str]]: + documents: list[str] = [] + filenames: list[str] = [] + for f in files: + if not Path(f).exists(): + _out(f"Warning: {f} does not exist") + continue + text = read_file(f) + if text.strip(): + documents.append(preprocess_text(text)) + filenames.append(Path(f).name) + else: + _out(f"Warning: {f} is empty or unreadable") + return documents, filenames + + +def _report_pairs( + documents: list[str], + filenames: list[str], + sim_matrix: list[list[float]], + threshold: float, +) -> list[tuple[int, int, float, str]]: + _out(f"{'Document Pair':<50} {'Similarity':>12}") + _out("-" * 62) + + suspicious_pairs: list[tuple[int, int, float, str]] = [] + for i in range(len(documents)): + for j in range(i + 1, len(documents)): + similarity = sim_matrix[i][j] + pair_name = f"{filenames[i]} <-> {filenames[j]}" + if similarity >= threshold: + suspicious_pairs.append((i, j, similarity, pair_name)) + _out(f"{pair_name:<50} {similarity:>10.1%} ⚠️") + else: + _out(f"{pair_name:<50} {similarity:>10.1%}") + + _out("-" * 62) + return suspicious_pairs + + +def _print_detailed_passages( + files: list[str], suspicious_pairs: list[tuple[int, int, float, str]] +) -> None: + _out("\n" + "=" * 60) + _out(" Detailed Similar Passages") + _out("=" * 60) + + for i, j, sim, pair_name in suspicious_pairs[:3]: + _out(f"\n{pair_name} ({sim:.1%} similar):") + _out("-" * 40) + + raw_docs = [read_file(files[i]), read_file(files[j])] + passages = find_similar_passages(raw_docs[0], raw_docs[1]) + + for s1, s2, psim in passages[:5]: + _out(f'\n[{psim:.0%}] Document 1: "{s1[:100]}..."') + _out(f' Document 2: "{s2[:100]}..."') + + +def main() -> int: + """Compare every pair of input documents and report suspicious pairs.""" + parser = argparse.ArgumentParser( + description="Text Plagiarism Checker - Compare documents for similarity" + ) + parser.add_argument("files", nargs="*", help="Files to compare") + parser.add_argument("--dir", "-d", help="Directory containing documents to compare") + parser.add_argument( + "--threshold", + "-t", + type=float, + default=0.3, + help="Similarity threshold for flagging (0-1, default: 0.3)", + ) + parser.add_argument( + "--detailed", "-v", action="store_true", help="Show detailed similar passages" + ) + + args = parser.parse_args() + + files = _collect_files(args) + if len(files) < MIN_DOCUMENTS: + _out("Error: Need at least 2 files to compare") + parser.print_help() + return 1 + + _out(f"\n{'=' * 60}") + _out(f" Plagiarism Check - Analyzing {len(files)} documents") + _out(f"{'=' * 60}\n") + + documents, filenames = _read_documents(files) + if len(documents) < MIN_DOCUMENTS: + _out("Error: Not enough valid documents to compare") + return 1 + + _out("Computing document similarities...\n") + sim_matrix = compute_similarity_matrix(documents) + + suspicious_pairs = _report_pairs(documents, filenames, sim_matrix, args.threshold) + + if suspicious_pairs: + _out( + f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed " + f"{args.threshold:.0%} similarity threshold\n" + ) + if args.detailed: + _print_detailed_passages(files, suspicious_pairs) + else: + _out(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold") + + _out("\n" + "=" * 60) + _out(" Analysis complete") + _out("=" * 60 + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh index 3956090..05deda5 100755 --- a/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh +++ b/linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh @@ -13,12 +13,6 @@ set -euo pipefail INSTALL_DIR="${HOME}/.local/share/plagiarism-tools" VENV_DIR="${HOME}/.local/share/plagiarism-venv" -echo "==============================================" -echo " Open Source Plagiarism Detection Installer" -echo " For Academic Text (Theses, Papers, etc.)" -echo "==============================================" -echo "" - # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' @@ -29,506 +23,72 @@ success() { echo -e "${GREEN}✓ $1${NC}"; } warn() { echo -e "${YELLOW}⚠ $1${NC}"; } error() { echo -e "${RED}✗ $1${NC}"; } -# Create installation directory -mkdir -p "$INSTALL_DIR" - -# ------------------------------------------------------------------------------ -# 1. Python-based NLP Plagiarism Detection Environment -# ------------------------------------------------------------------------------ -echo "" -echo "=== 1. Installing Python NLP-based Plagiarism Tools ===" - -# Check for Python 3 -if ! command -v python3 &>/dev/null; then - error "Python 3 is required but not installed." - exit 1 -fi - -# Create virtual environment -if [ ! -d "$VENV_DIR" ]; then - echo "Creating Python virtual environment..." - python3 -m venv "$VENV_DIR" - success "Virtual environment created at $VENV_DIR" -else - warn "Virtual environment already exists at $VENV_DIR" -fi - -# Activate and install packages -# Created at runtime by `python -m venv` above, so there is nothing on disk -# for the linter to follow at check time. -# shellcheck disable=SC1091 -source "$VENV_DIR/bin/activate" - -echo "Installing Python packages for text similarity detection..." -pip install --upgrade pip - -pip install --progress-bar on \ - scikit-learn \ - nltk \ - spacy \ - gensim \ - numpy \ - pandas \ - python-docx \ - PyPDF2 \ - beautifulsoup4 \ - lxml \ - textdistance \ - fuzzywuzzy \ - python-Levenshtein - -success "Python NLP packages installed" - -# Download NLTK data -echo "Downloading NLTK data (stopwords, punkt tokenizer)..." -python3 "$(dirname "$0")/download_nltk_data.py" -success "NLTK data downloaded" - -# Download spaCy English model (small) -echo "Downloading spaCy English model..." -python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm" -success "spaCy model installed" - -# Create a simple plagiarism checker script -cat >"$INSTALL_DIR/check_plagiarism.py" <<'PYEOF' -#!/usr/bin/env python3 -""" -Simple Text Plagiarism Checker -Compares documents using multiple similarity algorithms. - -Usage: - python check_plagiarism.py file1.txt file2.txt [file3.txt ...] - python check_plagiarism.py --dir /path/to/documents/ -""" - -import argparse -import os -import sys -from pathlib import Path -from typing import List, Tuple - -import numpy as np -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.metrics.pairwise import cosine_similarity -import nltk -from nltk.corpus import stopwords -from nltk.tokenize import word_tokenize, sent_tokenize - -# Ensure NLTK data is available -try: - stopwords.words('english') -except LookupError: - nltk.download('stopwords', quiet=True) - nltk.download('punkt', quiet=True) - - -def read_file(filepath: str) -> str: - """Read text from various file formats.""" - path = Path(filepath) - suffix = path.suffix.lower() - - if suffix == '.pdf': - try: - from PyPDF2 import PdfReader - reader = PdfReader(filepath) - return ' '.join(page.extract_text() or '' for page in reader.pages) - except ImportError: - print("Warning: PyPDF2 not installed, cannot read PDF files") - return "" - elif suffix == '.docx': - try: - from docx import Document - doc = Document(filepath) - return ' '.join(para.text for para in doc.paragraphs) - except ImportError: - print("Warning: python-docx not installed, cannot read DOCX files") - return "" - else: - # Assume plain text - with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: - return f.read() - - -def preprocess_text(text: str) -> str: - """Clean and preprocess text for comparison.""" - # Lowercase - text = text.lower() - # Tokenize and remove stopwords - try: - stop_words = set(stopwords.words('english')) - words = word_tokenize(text) - words = [w for w in words if w.isalnum() and w not in stop_words] - return ' '.join(words) - except Exception: - # Fallback: simple preprocessing - return ' '.join(text.split()) - - -def compute_similarity_matrix(documents: List[str]) -> np.ndarray: - """Compute TF-IDF cosine similarity matrix.""" - vectorizer = TfidfVectorizer( - ngram_range=(1, 3), # Use unigrams, bigrams, trigrams - min_df=1, - max_df=0.95 - ) - tfidf_matrix = vectorizer.fit_transform(documents) - return cosine_similarity(tfidf_matrix) - - -def find_similar_passages(text1: str, text2: str, min_words: int = 5) -> List[Tuple[str, str, float]]: - """Find similar sentence-level passages between two texts.""" - sentences1 = sent_tokenize(text1) - sentences2 = sent_tokenize(text2) - - if not sentences1 or not sentences2: - return [] - - # Filter short sentences - sentences1 = [s for s in sentences1 if len(s.split()) >= min_words] - sentences2 = [s for s in sentences2 if len(s.split()) >= min_words] - - if not sentences1 or not sentences2: - return [] - - all_sentences = sentences1 + sentences2 - preprocessed = [preprocess_text(s) for s in all_sentences] - - try: - vectorizer = TfidfVectorizer(ngram_range=(1, 2)) - tfidf_matrix = vectorizer.fit_transform(preprocessed) - except ValueError: - return [] - - n1 = len(sentences1) - similarities = [] - - for i, s1 in enumerate(sentences1): - for j, s2 in enumerate(sentences2): - sim = cosine_similarity( - tfidf_matrix[i:i+1], - tfidf_matrix[n1+j:n1+j+1] - )[0][0] - if sim > 0.5: # Threshold for suspicious similarity - similarities.append((s1, s2, sim)) - - return sorted(similarities, key=lambda x: x[2], reverse=True) - - -def main(): - parser = argparse.ArgumentParser( - description='Text Plagiarism Checker - Compare documents for similarity' - ) - parser.add_argument('files', nargs='*', help='Files to compare') - parser.add_argument('--dir', '-d', help='Directory containing documents to compare') - parser.add_argument('--threshold', '-t', type=float, default=0.3, - help='Similarity threshold for flagging (0-1, default: 0.3)') - parser.add_argument('--detailed', '-v', action='store_true', - help='Show detailed similar passages') - - args = parser.parse_args() - - # Collect files - files = [] - if args.files: - files.extend(args.files) - if args.dir: - dir_path = Path(args.dir) - for ext in ['*.txt', '*.pdf', '*.docx', '*.md', '*.tex']: - files.extend(str(f) for f in dir_path.glob(ext)) - - if len(files) < 2: - print("Error: Need at least 2 files to compare") - parser.print_help() - sys.exit(1) - - print(f"\n{'='*60}") - print(f" Plagiarism Check - Analyzing {len(files)} documents") - print(f"{'='*60}\n") - - # Read and preprocess documents - documents = [] - filenames = [] - for f in files: - if os.path.exists(f): - text = read_file(f) - if text.strip(): - documents.append(preprocess_text(text)) - filenames.append(os.path.basename(f)) - else: - print(f"Warning: {f} is empty or unreadable") - else: - print(f"Warning: {f} does not exist") - - if len(documents) < 2: - print("Error: Not enough valid documents to compare") - sys.exit(1) - - # Compute similarity - print("Computing document similarities...\n") - sim_matrix = compute_similarity_matrix(documents) - - # Report results - print(f"{'Document Pair':<50} {'Similarity':>12}") - print("-" * 62) - - suspicious_pairs = [] - for i in range(len(documents)): - for j in range(i + 1, len(documents)): - similarity = sim_matrix[i][j] - pair_name = f"{filenames[i]} <-> {filenames[j]}" - - if similarity >= args.threshold: - suspicious_pairs.append((i, j, similarity, pair_name)) - print(f"{pair_name:<50} {similarity:>10.1%} ⚠️") - else: - print(f"{pair_name:<50} {similarity:>10.1%}") - - print("-" * 62) - - # Summary - if suspicious_pairs: - print(f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed {args.threshold:.0%} similarity threshold\n") - - if args.detailed: - print("\n" + "="*60) - print(" Detailed Similar Passages") - print("="*60) - - for i, j, sim, pair_name in suspicious_pairs[:3]: # Limit to top 3 - print(f"\n{pair_name} ({sim:.1%} similar):") - print("-" * 40) - - raw_docs = [read_file(files[i]), read_file(files[j])] - passages = find_similar_passages(raw_docs[0], raw_docs[1]) - - for s1, s2, psim in passages[:5]: # Top 5 passages - print(f"\n[{psim:.0%}] Document 1: \"{s1[:100]}...\"") - print(f" Document 2: \"{s2[:100]}...\"") - else: - print(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold") - - print("\n" + "="*60) - print(" Analysis complete") - print("="*60 + "\n") - - -if __name__ == '__main__': - main() -PYEOF - -chmod +x "$INSTALL_DIR/check_plagiarism.py" -success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py" - -# Create convenience wrapper -mkdir -p "$HOME/.local/bin" -cat >"$HOME/.local/bin/plagcheck" </dev/null; then - # Clone a text-based similarity tool - git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$SHERLOCK_DIR" 2>/dev/null || { - warn "Could not clone sherlock-py, trying alternative..." - # Alternative: Create a simple n-gram based sherlock - mkdir -p "$SHERLOCK_DIR" - cat >"$SHERLOCK_DIR/sherlock.py" <<'SHERLOCKEOF' -#!/usr/bin/env python3 -""" -Sherlock - Simple text plagiarism detector using n-gram fingerprinting. -Based on the original Sherlock algorithm. -""" - -import argparse -import hashlib -import os -import sys -from collections import defaultdict -from pathlib import Path - - -def tokenize(text: str) -> list: - """Simple word tokenization.""" - return [w.lower() for w in text.split() if w.isalnum()] - - -def get_ngrams(tokens: list, n: int = 3) -> list: - """Generate n-grams from token list.""" - return [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)] - - -def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set: - """Create document fingerprint using sampled n-gram hashes.""" - tokens = tokenize(text) - ngrams = get_ngrams(tokens, ngram_size) - - fingerprints = set() - for i, ng in enumerate(ngrams): - if i % sample_rate == 0: # Sample every nth n-gram - h = hashlib.md5(''.join(ng).encode()).hexdigest()[:8] - fingerprints.add(h) - - return fingerprints - - -def compare_documents(fp1: set, fp2: set) -> float: - """Jaccard similarity between fingerprints.""" - if not fp1 or not fp2: - return 0.0 - intersection = len(fp1 & fp2) - union = len(fp1 | fp2) - return intersection / union if union > 0 else 0.0 - - -def read_document(filepath: str) -> str: - """Read document content.""" - with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: - return f.read() - - -def main(): - parser = argparse.ArgumentParser(description='Sherlock - Text Plagiarism Detector') - parser.add_argument('files', nargs='+', help='Files to compare') - parser.add_argument('--ngram', '-n', type=int, default=3, help='N-gram size (default: 3)') - parser.add_argument('--threshold', '-t', type=float, default=0.1, help='Similarity threshold') - - args = parser.parse_args() - - if len(args.files) < 2: - print("Need at least 2 files to compare") - sys.exit(1) - - # Read and fingerprint documents - docs = {} - for f in args.files: - if os.path.exists(f): - text = read_document(f) - docs[f] = fingerprint(text, args.ngram) - - print(f"\nSherlock Plagiarism Analysis") - print("=" * 50) - - # Compare all pairs - files = list(docs.keys()) - for i in range(len(files)): - for j in range(i + 1, len(files)): - sim = compare_documents(docs[files[i]], docs[files[j]]) - name1 = os.path.basename(files[i]) - name2 = os.path.basename(files[j]) - flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else "" - print(f"{name1} <-> {name2}: {sim:.1%}{flag}") - - print("=" * 50) - - -if __name__ == '__main__': - main() -SHERLOCKEOF - chmod +x "$SHERLOCK_DIR/sherlock.py" - } - success "Sherlock installed at $SHERLOCK_DIR" - else - warn "Git not available, skipping Sherlock installation" - fi -else - warn "Sherlock already installed at $SHERLOCK_DIR" -fi - -# ------------------------------------------------------------------------------ -# 3. Ferret (Java-based) - Optional -# ------------------------------------------------------------------------------ -echo "" -echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ===" - -if command -v java &>/dev/null; then - FERRET_DIR="$INSTALL_DIR/ferret" - if [ ! -d "$FERRET_DIR" ]; then - mkdir -p "$FERRET_DIR" - echo "Ferret is a Java-based tool from University of Hertfordshire." - echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/" - echo "Place JAR file in: $FERRET_DIR" - warn "Ferret requires manual download (academic license)" - fi -else - warn "Java not installed, skipping Ferret" -fi - -# ------------------------------------------------------------------------------ -# 4. WCopyfind via Wine (Optional) -# ------------------------------------------------------------------------------ -echo "" -echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ===" - -if command -v wine &>/dev/null; then - echo "Wine is available. WCopyfind can be run via Wine." - echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" - echo "Run with: wine /path/to/WCopyfind.exe" - warn "WCopyfind requires manual download" -else - echo "Wine not installed. To use WCopyfind:" - echo " 1. Install wine: sudo apt install wine (or equivalent)" - echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" - warn "WCopyfind skipped (Wine not available)" -fi - -# ------------------------------------------------------------------------------ -# Summary -# ------------------------------------------------------------------------------ -echo "" -echo "==============================================" -echo " Installation Complete!" -echo "==============================================" -echo "" -echo "Installed tools:" -echo "" -echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)" -echo " Usage: plagcheck file1.txt file2.txt" -echo " plagcheck --dir /path/to/documents/ --detailed" -echo " Location: $INSTALL_DIR/check_plagiarism.py" -echo "" -echo "2. Sherlock (n-gram fingerprinting)" -echo " Location: $SHERLOCK_DIR/sherlock.py" -echo "" -echo "3. Python virtual environment with NLP libraries:" -echo " - scikit-learn (TF-IDF, cosine similarity)" -echo " - NLTK (tokenization, stopwords)" -echo " - spaCy (NLP processing)" -echo " - gensim (document similarity)" -echo " - textdistance, fuzzywuzzy (string matching)" -echo " Activate with: source $VENV_DIR/bin/activate" -echo "" -echo "Quick Start:" -echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed" -echo " plagcheck --dir ./student_papers/ --threshold 0.4" -echo "" -echo "Note: Ensure ~/.local/bin is in your PATH:" -# Printed for the user to paste into their shell rc: $HOME and $PATH must -# stay literal rather than resolve to this machine's values. -# shellcheck disable=SC2016 -echo ' export PATH="$HOME/.local/bin:$PATH"' -echo "" -echo "==============================================" - -# Add to PATH reminder -if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then - warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:" +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +# shellcheck source=lib/plagiarism_python.sh +source "$SCRIPT_DIR/lib/plagiarism_python.sh" +# shellcheck source=lib/plagiarism_sherlock.sh +source "$SCRIPT_DIR/lib/plagiarism_sherlock.sh" +# shellcheck source=lib/plagiarism_optional.sh +source "$SCRIPT_DIR/lib/plagiarism_optional.sh" + +print_summary() { + echo "" + echo "==============================================" + echo " Installation Complete!" + echo "==============================================" + echo "" + echo "Installed tools:" + echo "" + echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)" + echo " Usage: plagcheck file1.txt file2.txt" + echo " plagcheck --dir /path/to/documents/ --detailed" + echo " Location: $INSTALL_DIR/check_plagiarism.py" + echo "" + echo "2. Sherlock (n-gram fingerprinting)" + echo " Location: $INSTALL_DIR/sherlock/sherlock.py" + echo "" + echo "3. Python virtual environment with NLP libraries:" + echo " - scikit-learn (TF-IDF, cosine similarity)" + echo " - NLTK (tokenization, stopwords)" + echo " - spaCy (NLP processing)" + echo " - gensim (document similarity)" + echo " - textdistance, fuzzywuzzy (string matching)" + echo " Activate with: source $VENV_DIR/bin/activate" + echo "" + echo "Quick Start:" + echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed" + echo " plagcheck --dir ./student_papers/ --threshold 0.4" + echo "" + echo "Note: Ensure ~/.local/bin is in your PATH:" # Printed for the user to paste into their shell rc: $HOME and $PATH must # stay literal rather than resolve to this machine's values. # shellcheck disable=SC2016 echo ' export PATH="$HOME/.local/bin:$PATH"' -fi + echo "" + echo "==============================================" + + if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:" + # shellcheck disable=SC2016 + echo ' export PATH="$HOME/.local/bin:$PATH"' + fi +} + +main() { + echo "==============================================" + echo " Open Source Plagiarism Detection Installer" + echo " For Academic Text (Theses, Papers, etc.)" + echo "==============================================" + echo "" + + mkdir -p "$INSTALL_DIR" + + install_python_nlp_tools "$SCRIPT_DIR" + install_sherlock "$SCRIPT_DIR" + check_ferret + check_wcopyfind + + print_summary +} + +main "$@" diff --git a/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh b/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh new file mode 100755 index 0000000..d6db2d4 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Stages 3-4: optional plagiarism tools that need a manual download. +# +# Sourced by install_plagiarism_tools.sh; split out to keep it under the +# 250-line cap. Sourced rather than run, so it inherits the caller's strict +# mode and the color/log helpers and INSTALL_DIR global defined above the +# source line. + +# Ferret (Java-based) - Optional +check_ferret() { + echo "" + echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ===" + + if ! command -v java &>/dev/null; then + warn "Java not installed, skipping Ferret" + return + fi + + local ferret_dir="$INSTALL_DIR/ferret" + if [ -d "$ferret_dir" ]; then + return + fi + + mkdir -p "$ferret_dir" + echo "Ferret is a Java-based tool from University of Hertfordshire." + echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/" + echo "Place JAR file in: $ferret_dir" + warn "Ferret requires manual download (academic license)" +} + +# WCopyfind via Wine (Optional) +check_wcopyfind() { + echo "" + echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ===" + + if command -v wine &>/dev/null; then + echo "Wine is available. WCopyfind can be run via Wine." + echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + echo "Run with: wine /path/to/WCopyfind.exe" + warn "WCopyfind requires manual download" + return + fi + + echo "Wine not installed. To use WCopyfind:" + echo " 1. Install wine: sudo apt install wine (or equivalent)" + echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/" + warn "WCopyfind skipped (Wine not available)" +} diff --git a/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh b/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh new file mode 100755 index 0000000..0eda6da --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Stage 1: Python NLP-based plagiarism detection environment. +# +# Sourced by install_plagiarism_tools.sh; split out to keep it under the +# 250-line cap. Sourced rather than run, so it inherits the caller's strict +# mode and the color/log helpers and INSTALL_DIR/VENV_DIR globals defined +# above the source line. + +# Create the venv, pip-install the NLP stack, and install check_plagiarism.py +# (plus the plagiarism_similarity.py module it imports) and its "plagcheck" +# wrapper. $1 is the directory install_plagiarism_tools.sh lives in, so both +# scripts can be copied from beside it. +install_python_nlp_tools() { + local script_dir="$1" + + echo "" + echo "=== 1. Installing Python NLP-based Plagiarism Tools ===" + + if ! command -v python3 &>/dev/null; then + error "Python 3 is required but not installed." + exit 1 + fi + + if [ ! -d "$VENV_DIR" ]; then + echo "Creating Python virtual environment..." + python3 -m venv "$VENV_DIR" + success "Virtual environment created at $VENV_DIR" + else + warn "Virtual environment already exists at $VENV_DIR" + fi + + # Created at runtime by `python -m venv` above, so there is nothing on disk + # for the linter to follow at check time. + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + + echo "Installing Python packages for text similarity detection..." + pip install --upgrade pip + + pip install --progress-bar on \ + scikit-learn \ + nltk \ + spacy \ + gensim \ + numpy \ + pandas \ + python-docx \ + PyPDF2 \ + beautifulsoup4 \ + lxml \ + textdistance \ + fuzzywuzzy \ + python-Levenshtein + + success "Python NLP packages installed" + + echo "Downloading NLTK data (stopwords, punkt tokenizer)..." + python3 "$script_dir/download_nltk_data.py" + success "NLTK data downloaded" + + echo "Downloading spaCy English model..." + python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm" + success "spaCy model installed" + + install -m 755 "$script_dir/check_plagiarism.py" "$INSTALL_DIR/check_plagiarism.py" + install -m 644 "$script_dir/plagiarism_similarity.py" "$INSTALL_DIR/plagiarism_similarity.py" + success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py" + + mkdir -p "$HOME/.local/bin" + cat >"$HOME/.local/bin/plagcheck" </dev/null; then + warn "Git not available, skipping Sherlock installation" + return + fi + + # There are several Sherlock implementations; using a popular Python one + if git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$sherlock_dir" 2>/dev/null; then + success "Sherlock installed at $sherlock_dir" + return + fi + + warn "Could not clone sherlock-py, trying alternative..." + mkdir -p "$sherlock_dir" + install -m 755 "$script_dir/sherlock.py" "$sherlock_dir/sherlock.py" + success "Sherlock installed at $sherlock_dir" +} diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh b/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh new file mode 100755 index 0000000..d57d633 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# lib/tests/plagiarism_harness.sh — fake python3/pip/git/java/wine behind the +# install_plagiarism_tools.sh split. +# +# Sourced, not executed. Every external tool the installer calls is a shim on +# PATH that records its invocation into $DEV and does the minimum real +# filesystem work the calling code depends on (e.g. `python3 -m venv` must +# leave a `bin/activate` a real `source` can run). INSTALL_DIR, VENV_DIR and +# HOME all point into a throwaway tmpdir so nothing touches the real machine. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +UTILS_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +PASS=0 +FAIL=0 + +_t_pass() { + PASS=$((PASS + 1)) + printf ' OK: %s\n' "$1" +} + +_t_fail() { + FAIL=$((FAIL + 1)) + printf ' FAIL: %s\n' "$1" +} + +_t_eq() { + local want="$1" got="$2" what="$3" + if [[ "$got" == "$want" ]]; then + _t_pass "$what" + else + _t_fail "$what (want '${want}', got '${got}')" + fi +} + +TEST_TMPDIR="$(mktemp -d)" +trap 'rm -rf "${TEST_TMPDIR}"' EXIT + +readonly DEV="${TEST_TMPDIR}/device" +readonly FAKE_BIN="${TEST_TMPDIR}/fake_bin" +mkdir -p "${DEV}" "${FAKE_BIN}" + +# --- fake external tools ---------------------------------------------------- + +cat >"${FAKE_BIN}/python3" <<'PYSHIM' +#!/usr/bin/env bash +set -euo pipefail +DEV="${PLAGIARISM_TEST_DEV}" +printf '%s\n' "python3 $*" >>"${DEV}/calls" +if [[ "$1" == "-m" && "$2" == "venv" ]]; then + venv_dir="$3" + mkdir -p "${venv_dir}/bin" + cat >"${venv_dir}/bin/activate" <<'ACTEOF' +deactivate() { :; } +pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; } +ACTEOF + exit 0 +fi +if [[ "$1" == "-m" && "$2" == "spacy" ]]; then + [[ -f "${DEV}/fail_spacy" ]] && exit 1 + exit 0 +fi +# python3 [args...] — record which script ran and succeed. +printf '%s\n' "ran:$1" >>"${DEV}/scripts_run" +[[ -f "${DEV}/fail_$(basename "$1" .py)" ]] && exit 1 +exit 0 +PYSHIM +chmod +x "${FAKE_BIN}/python3" + +cat >"${FAKE_BIN}/git" <<'GITSHIM' +#!/usr/bin/env bash +set -euo pipefail +DEV="${PLAGIARISM_TEST_DEV}" +printf '%s\n' "git $*" >>"${DEV}/calls" +if [[ "$1" == "clone" ]]; then + [[ -f "${DEV}/fail_clone" ]] && exit 1 + dest="${*: -1}" + mkdir -p "${dest}" + exit 0 +fi +exit 0 +GITSHIM +chmod +x "${FAKE_BIN}/git" + +cat >"${FAKE_BIN}/java" <<'JAVASHIM' +#!/usr/bin/env bash +exit 0 +JAVASHIM +chmod +x "${FAKE_BIN}/java" + +cat >"${FAKE_BIN}/wine" <<'WINESHIM' +#!/usr/bin/env bash +exit 0 +WINESHIM +chmod +x "${FAKE_BIN}/wine" + +export PLAGIARISM_TEST_DEV="${DEV}" +export PATH="${FAKE_BIN}:${PATH}" + +# --- subject under test ----------------------------------------------------- + +HOME="${TEST_TMPDIR}/home" +INSTALL_DIR="${TEST_TMPDIR}/install" +VENV_DIR="${TEST_TMPDIR}/venv" +mkdir -p "${HOME}" "${INSTALL_DIR}" + +success() { :; } +warn() { printf '%s\n' "$*" >>"${DEV}/warnings"; } +error() { printf '%s\n' "$*" >>"${DEV}/errors"; } + +# shellcheck source=../plagiarism_python.sh +. "${UTILS_DIR}/lib/plagiarism_python.sh" +# shellcheck source=../plagiarism_sherlock.sh +. "${UTILS_DIR}/lib/plagiarism_sherlock.sh" +# shellcheck source=../plagiarism_optional.sh +. "${UTILS_DIR}/lib/plagiarism_optional.sh" + +# reset_state — wipe INSTALL_DIR/VENV_DIR/DEV records between test groups so +# each group starts from "nothing installed yet". +reset_state() { + rm -rf "${INSTALL_DIR}" "${VENV_DIR}" "${DEV:?}/calls" "${DEV}/scripts_run" \ + "${DEV}/warnings" "${DEV}/errors" 2>/dev/null || true + mkdir -p "${INSTALL_DIR}" + : >"${DEV}/calls" + : >"${DEV}/scripts_run" + : >"${DEV}/warnings" + : >"${DEV}/errors" +} +reset_state diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh new file mode 100755 index 0000000..a4af8f4 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_install_plagiarism_tools.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# End-to-end test for install_plagiarism_tools.sh's main()/print_summary(), +# which stay in the entry script rather than a lib. Runs the real script as a +# subprocess against the same fake python3/pip/git/java/wine as the lib tests, +# with HOME pointed at a throwaway tmpdir so INSTALL_DIR/VENV_DIR (derived +# from $HOME inside the script) land there too. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +readonly ENTRY_SCRIPT="${UTILS_DIR}/install_plagiarism_tools.sh" + +# The entry script derives INSTALL_DIR/VENV_DIR from $HOME itself (a plain +# assignment near its top), independently of the lib tests' INSTALL_DIR/ +# VENV_DIR harness variables. Running it as a real subprocess against the +# harness's $HOME means its paths land under $HOME, not under the harness's +# separate INSTALL_DIR/VENV_DIR -- so assertions below re-derive the same +# paths the script itself computes, rather than reusing the harness ones. +readonly ENTRY_INSTALL_DIR="${HOME}/.local/share/plagiarism-tools" +readonly ENTRY_VENV_DIR="${HOME}/.local/share/plagiarism-venv" + +# --- full run, PATH already in ~/.local/bin ------------------------------- + +mkdir -p "${HOME}/.local/bin" +OUT_FULL="${TEST_TMPDIR}/out_full.log" +PATH="${FAKE_BIN}:${HOME}/.local/bin:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_FULL}" 2>&1 +_status=$? + +if [[ "${_status}" -eq 0 ]]; then + _t_pass "install_plagiarism_tools.sh exits zero on a full successful run" +else + _t_fail "install_plagiarism_tools.sh should exit zero (log: $(cat "${OUT_FULL}"))" +fi + +if grep -q "Open Source Plagiarism Detection Installer" "${OUT_FULL}"; then + _t_pass "main prints the installer banner" +else + _t_fail "main should print the installer banner" +fi + +if grep -q "=== 1. Installing Python NLP-based Plagiarism Tools ===" "${OUT_FULL}"; then + _t_pass "main runs stage 1 (Python NLP tools)" +else + _t_fail "main should run stage 1" +fi + +if grep -q "=== 2. Installing Sherlock Text Plagiarism Detector ===" "${OUT_FULL}"; then + _t_pass "main runs stage 2 (Sherlock)" +else + _t_fail "main should run stage 2" +fi + +if grep -q "=== 3. Checking for Ferret" "${OUT_FULL}"; then + _t_pass "main runs stage 3 (Ferret)" +else + _t_fail "main should run stage 3" +fi + +if grep -q "=== 4. WCopyfind Information" "${OUT_FULL}"; then + _t_pass "main runs stage 4 (WCopyfind)" +else + _t_fail "main should run stage 4" +fi + +if grep -q "Installation Complete!" "${OUT_FULL}"; then + _t_pass "main prints the summary" +else + _t_fail "main should print the summary" +fi + +if grep -q "Location: ${ENTRY_INSTALL_DIR}/check_plagiarism.py" "${OUT_FULL}"; then + _t_pass "the summary reports the checker's installed location" +else + _t_fail "the summary should report the checker's installed location" +fi + +if grep -q "Location: ${ENTRY_INSTALL_DIR}/sherlock/sherlock.py" "${OUT_FULL}"; then + _t_pass "the summary reports sherlock's installed location" +else + _t_fail "the summary should report sherlock's installed location" +fi + +if grep -q "Activate with: source ${ENTRY_VENV_DIR}/bin/activate" "${OUT_FULL}"; then + _t_pass "the summary reports the venv activation command" +else + _t_fail "the summary should report the venv activation command" +fi + +# ~/.local/bin was already on PATH for this run, so the "add it to your PATH" +# nudge must not repeat in the summary. +if grep -q "Add ~/.local/bin to your PATH by adding this" "${OUT_FULL}"; then + _t_fail "the summary should not repeat the PATH nudge when ~/.local/bin is already on PATH" +else + _t_pass "the summary skips the PATH nudge when ~/.local/bin is already on PATH" +fi + +# --- ~/.local/bin missing from PATH: the summary nudges once -------------- + +reset_state +rm -rf "${HOME}/.local" +OUT_NOPATH="${TEST_TMPDIR}/out_nopath.log" +PATH="${FAKE_BIN}:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_NOPATH}" 2>&1 + +if grep -qc "Add ~/.local/bin to your PATH by adding this" "${OUT_NOPATH}"; then + _t_pass "the summary nudges to add ~/.local/bin to PATH when it is missing" +else + _t_fail "the summary should nudge to add ~/.local/bin to PATH when it is missing" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh new file mode 100755 index 0000000..87b703f --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_optional.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Unit tests for lib/plagiarism_optional.sh (Ferret/WCopyfind availability +# checks) against the fake java/wine in plagiarism_harness.sh. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +_no_tool_path() { + local skip="$1" + local dir="${TEST_TMPDIR}/no_${skip}_bin" + mkdir -p "${dir}" + local tool + for tool in python3 git java wine cat chmod mkdir; do + [[ "${tool}" == "${skip}" ]] && continue + [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${dir}/${tool}" + done + printf '%s' "${dir}" +} + +# --- check_ferret: java available ----------------------------------------- + +check_ferret >/tmp/plagiarism_optional_test.log 2>&1 + +if [[ -d "${INSTALL_DIR}/ferret" ]]; then + _t_pass "check_ferret creates the ferret directory when java is available" +else + _t_fail "check_ferret should create ${INSTALL_DIR}/ferret when java is available" +fi + +if grep -q "requires manual download" "${DEV}/warnings"; then + _t_pass "check_ferret explains that Ferret needs a manual download" +else + _t_fail "check_ferret should explain that Ferret needs a manual download" +fi + +# check_ferret creates the directory as a marker; a second run must not +# re-print the manual-download instructions once it already exists. +: >"${DEV}/warnings" +check_ferret >/tmp/plagiarism_optional_test.log 2>&1 +if grep -q "requires manual download" "${DEV}/warnings"; then + _t_fail "check_ferret should not repeat instructions once the ferret dir already exists" +else + _t_pass "check_ferret is a no-op once the ferret directory already exists" +fi + +# --- check_ferret: java unavailable --------------------------------------- + +reset_state +no_java_path="$(_no_tool_path java)" +PATH="${no_java_path}" check_ferret >/dev/null 2>&1 + +if [[ -d "${INSTALL_DIR}/ferret" ]]; then + _t_fail "check_ferret should not create the ferret directory without java" +else + _t_pass "check_ferret skips Ferret entirely when java is unavailable" +fi + +if grep -q "Java not installed" "${DEV}/warnings"; then + _t_pass "check_ferret warns when java is unavailable" +else + _t_fail "check_ferret should warn when java is unavailable" +fi + +# --- check_wcopyfind: wine available --------------------------------------- + +: >"${DEV}/warnings" +check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1 + +if grep -q "Wine is available" /tmp/plagiarism_optional_test.log; then + _t_pass "check_wcopyfind reports Wine as available" +else + _t_fail "check_wcopyfind should report Wine as available" +fi + +if grep -q "requires manual download" "${DEV}/warnings"; then + _t_pass "check_wcopyfind warns that WCopyfind needs a manual download" +else + _t_fail "check_wcopyfind should warn that WCopyfind needs a manual download" +fi + +# --- check_wcopyfind: wine unavailable -------------------------------------- + +: >"${DEV}/warnings" +no_wine_path="$(_no_tool_path wine)" +PATH="${no_wine_path}" check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1 + +if grep -q "Wine not installed" /tmp/plagiarism_optional_test.log; then + _t_pass "check_wcopyfind reports Wine as unavailable" +else + _t_fail "check_wcopyfind should report Wine as unavailable" +fi + +if grep -q "skipped (Wine not available)" "${DEV}/warnings"; then + _t_pass "check_wcopyfind warns that WCopyfind was skipped" +else + _t_fail "check_wcopyfind should warn that WCopyfind was skipped" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh new file mode 100755 index 0000000..e913e34 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Unit tests for lib/plagiarism_python.sh against the fake python3/pip in +# plagiarism_harness.sh. No real network or venv is needed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +# --- happy path --------------------------------------------------------- + +install_python_nlp_tools "${UTILS_DIR}" >/tmp/plagiarism_python_test.log 2>&1 +_status=$? +if [[ "${_status}" -eq 0 ]]; then + _t_pass "install_python_nlp_tools succeeds against a working toolchain" +else + _t_fail "install_python_nlp_tools should succeed (log: $(cat /tmp/plagiarism_python_test.log))" +fi + +if [[ -d "${VENV_DIR}" ]]; then + _t_pass "install_python_nlp_tools creates the virtual environment" +else + _t_fail "install_python_nlp_tools should create ${VENV_DIR}" +fi + +if grep -q '^python3 -m venv' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools invokes python3 -m venv" +else + _t_fail "install_python_nlp_tools should invoke python3 -m venv" +fi + +if grep -q 'pip install --upgrade pip' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools upgrades pip" +else + _t_fail "install_python_nlp_tools should upgrade pip" +fi + +if grep -q 'scikit-learn' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools installs the NLP package set" +else + _t_fail "install_python_nlp_tools should install scikit-learn and friends" +fi + +if grep -q 'ran:.*download_nltk_data.py' "${DEV}/scripts_run"; then + _t_pass "install_python_nlp_tools runs download_nltk_data.py" +else + _t_fail "install_python_nlp_tools should run download_nltk_data.py" +fi + +if grep -q '^python3 -m spacy download' "${DEV}/calls"; then + _t_pass "install_python_nlp_tools downloads the spaCy model" +else + _t_fail "install_python_nlp_tools should download the spaCy model" +fi + +if [[ -x "${INSTALL_DIR}/check_plagiarism.py" ]]; then + _t_pass "install_python_nlp_tools installs check_plagiarism.py, executable" +else + _t_fail "install_python_nlp_tools should install an executable check_plagiarism.py" +fi + +if [[ -f "${INSTALL_DIR}/plagiarism_similarity.py" ]]; then + _t_pass "install_python_nlp_tools installs plagiarism_similarity.py alongside it" +else + _t_fail "install_python_nlp_tools should install plagiarism_similarity.py -- \ +check_plagiarism.py imports it, so a missing copy breaks at runtime, not install time" +fi + +if [[ -x "${HOME}/.local/bin/plagcheck" ]]; then + _t_pass "install_python_nlp_tools creates an executable plagcheck wrapper" +else + _t_fail "install_python_nlp_tools should create an executable plagcheck wrapper" +fi + +if grep -q "${VENV_DIR}/bin/activate" "${HOME}/.local/bin/plagcheck"; then + _t_pass "plagcheck wrapper activates the venv it was installed against" +else + _t_fail "plagcheck wrapper should reference ${VENV_DIR}/bin/activate" +fi + +if grep -q "${INSTALL_DIR}/check_plagiarism.py" "${HOME}/.local/bin/plagcheck"; then + _t_pass "plagcheck wrapper points at the installed checker script" +else + _t_fail "plagcheck wrapper should point at ${INSTALL_DIR}/check_plagiarism.py" +fi + +# --- venv already exists: warn, don't recreate -------------------------- + +reset_state +mkdir -p "${VENV_DIR}/bin" +cat >"${VENV_DIR}/bin/activate" <<'ACTEOF' +deactivate() { :; } +pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; } +ACTEOF +: >"${DEV}/calls" +install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1 +if grep -q '^python3 -m venv' "${DEV}/calls"; then + _t_fail "install_python_nlp_tools should not recreate an existing venv" +else + _t_pass "install_python_nlp_tools skips venv creation when one already exists" +fi + +# --- python3 missing: fatal, not silently skipped ------------------------ + +reset_state +# A bare `rm -f "${FAKE_BIN}/python3"` does not hide python3 from +# `command -v`: pyenv's shim is still reachable further down the real PATH. +# Instead point PATH at a directory that shadows python3 with nothing before +# falling through to the rest of the fakes, so `command -v python3` fails. +NO_PYTHON_BIN="${TEST_TMPDIR}/no_python_bin" +mkdir -p "${NO_PYTHON_BIN}" +for tool in git java wine cat chmod mkdir; do + [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_PYTHON_BIN}/${tool}" +done +# install_python_nlp_tools calls `exit` (not `return`) on a missing python3, +# by design -- a partial install with no interpreter must abort the whole +# installer, not just this stage. Run it in a subshell so that exit only +# ends the subshell, not this test script. +if (PATH="${NO_PYTHON_BIN}" install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1); then + _t_fail "install_python_nlp_tools should exit non-zero without python3" +else + _t_pass "install_python_nlp_tools exits non-zero when python3 is missing" +fi +if grep -q "Python 3 is required" "${DEV}/errors"; then + _t_pass "install_python_nlp_tools reports the missing-python3 error" +else + _t_fail "install_python_nlp_tools should report the missing-python3 error" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh new file mode 100755 index 0000000..5abaa0e --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_sherlock.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Unit tests for lib/plagiarism_sherlock.sh against the fake git in +# plagiarism_harness.sh. No real network is needed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=plagiarism_harness.sh +. "${SCRIPT_DIR}/plagiarism_harness.sh" + +# --- upstream clone succeeds --------------------------------------------- + +install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if grep -q '^git clone' "${DEV}/calls"; then + _t_pass "install_sherlock clones the upstream sherlock-py tool" +else + _t_fail "install_sherlock should attempt a git clone" +fi + +if [[ -d "${INSTALL_DIR}/sherlock" ]]; then + _t_pass "install_sherlock creates the sherlock directory on a successful clone" +else + _t_fail "install_sherlock should create ${INSTALL_DIR}/sherlock" +fi + +if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then + _t_fail "a successful clone should not also install the bundled fallback sherlock.py" +else + _t_pass "a successful clone does not install the bundled fallback sherlock.py" +fi + +# --- already installed: no-op, warns ------------------------------------- + +: >"${DEV}/calls" +install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if grep -q '^git clone' "${DEV}/calls"; then + _t_fail "install_sherlock should not re-clone when already installed" +else + _t_pass "install_sherlock skips reinstalling when the directory already exists" +fi + +if grep -q "already installed" "${DEV}/warnings"; then + _t_pass "install_sherlock warns that sherlock is already installed" +else + _t_fail "install_sherlock should warn that sherlock is already installed" +fi + +# --- clone fails: falls back to the bundled sherlock.py ------------------- + +reset_state +touch "${DEV}/fail_clone" +install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then + _t_pass "install_sherlock falls back to the bundled sherlock.py when the clone fails" +else + _t_fail "install_sherlock should fall back to the bundled sherlock.py when the clone fails" +fi + +if [[ -x "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then + _t_pass "the bundled fallback sherlock.py is installed executable" +else + _t_fail "the bundled fallback sherlock.py should be installed executable" +fi + +if grep -q "Could not clone sherlock-py" "${DEV}/warnings"; then + _t_pass "install_sherlock warns when the clone fails, before falling back" +else + _t_fail "install_sherlock should warn when the clone fails" +fi + +# The installed fallback must be byte-identical to the source file: it is the +# one artifact the git-unavailable path can serve, so a stale copy would ship +# broken plagiarism detection silently. +if diff -q "${UTILS_DIR}/sherlock.py" "${INSTALL_DIR}/sherlock/sherlock.py" >/dev/null; then + _t_pass "the installed fallback sherlock.py matches the source file byte-for-byte" +else + _t_fail "the installed fallback sherlock.py should match ${UTILS_DIR}/sherlock.py" +fi + +# --- git not available: warns, does not attempt to install ---------------- + +reset_state +NO_GIT_BIN="${TEST_TMPDIR}/no_git_bin" +mkdir -p "${NO_GIT_BIN}" +for tool in python3 java wine cat chmod mkdir; do + [[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_GIT_BIN}/${tool}" +done +PATH="${NO_GIT_BIN}" install_sherlock "${UTILS_DIR}" >/dev/null 2>&1 + +if [[ -d "${INSTALL_DIR}/sherlock" ]]; then + _t_fail "install_sherlock should not create the sherlock directory without git" +else + _t_pass "install_sherlock skips installation entirely when git is unavailable" +fi + +if grep -q "Git not available" "${DEV}/warnings"; then + _t_pass "install_sherlock warns when git is unavailable" +else + _t_fail "install_sherlock should warn when git is unavailable" +fi + +printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}" +[[ "${FAIL}" -eq 0 ]] diff --git a/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py b/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py new file mode 100755 index 0000000..b11bee8 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/plagiarism_similarity.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Document reading and TF-IDF similarity scoring for check_plagiarism.py. + +Split out of `check_plagiarism.py` to keep it under the 250-line cap: this +module holds the comparison engine (file reading, text preprocessing, and the +TF-IDF/cosine-similarity computations), `check_plagiarism.py` keeps the CLI +and reporting. + +Its third-party imports (numpy, sklearn, nltk) are the plagiarism toolchain's +dependencies, not this repo's, so they are absent from `meta/requirements.txt` +on purpose. Importing them dynamically (rather than with a static `import +nltk`) keeps this module importable -- and lintable -- in an environment +that doesn't have them installed, matching `download_nltk_data.py`'s +extraction. +""" + +from __future__ import annotations + +import importlib +from pathlib import Path +import sys + +#: Cosine similarity above this, for a single sentence pair, is worth reporting +#: as a suspicious passage in --detailed mode. +PASSAGE_SIMILARITY_THRESHOLD = 0.5 + +nltk = importlib.import_module("nltk") +stopwords = importlib.import_module("nltk.corpus").stopwords +nltk_tokenize = importlib.import_module("nltk.tokenize") +sent_tokenize = nltk_tokenize.sent_tokenize +word_tokenize = nltk_tokenize.word_tokenize +TfidfVectorizer = importlib.import_module( + "sklearn.feature_extraction.text" +).TfidfVectorizer +cosine_similarity = importlib.import_module( + "sklearn.metrics.pairwise" +).cosine_similarity + +try: + stopwords.words("english") +except LookupError: + nltk.download("stopwords", quiet=True) + nltk.download("punkt", quiet=True) + + +def _out(line: str = "") -> None: + sys.stdout.write(line + "\n") + + +def _read_pdf(filepath: str) -> str: + try: + pypdf2 = importlib.import_module("PyPDF2") + except ImportError: + _out("Warning: PyPDF2 not installed, cannot read PDF files") + return "" + reader = pypdf2.PdfReader(filepath) + return " ".join(page.extract_text() or "" for page in reader.pages) + + +def _read_docx(filepath: str) -> str: + try: + docx = importlib.import_module("docx") + except ImportError: + _out("Warning: python-docx not installed, cannot read DOCX files") + return "" + doc = docx.Document(filepath) + return " ".join(para.text for para in doc.paragraphs) + + +def read_file(filepath: str) -> str: + """Read text from a .pdf, .docx, or plain-text file.""" + path = Path(filepath) + suffix = path.suffix.lower() + + if suffix == ".pdf": + return _read_pdf(filepath) + if suffix == ".docx": + return _read_docx(filepath) + with path.open(encoding="utf-8", errors="ignore") as f: + return f.read() + + +def preprocess_text(text: str) -> str: + """Lowercase, tokenize, and strip stopwords/non-alphanumerics.""" + text = text.lower() + try: + stop_words = set(stopwords.words("english")) + words = word_tokenize(text) + words = [w for w in words if w.isalnum() and w not in stop_words] + return " ".join(words) + except LookupError: + return " ".join(text.split()) + + +def compute_similarity_matrix(documents: list[str]) -> list[list[float]]: + """Compute the TF-IDF cosine similarity matrix for a set of documents. + + Returns a plain nested list, not the numpy array `cosine_similarity` + produces: numpy is a plagiarism-toolchain dependency the installer + pip-installs at runtime, deliberately absent from `meta/requirements.txt`, + so mypy cannot follow it into this repo's type-checked surface + (`no-any-unimported`). Converting at this boundary keeps every other + signature in the module concretely typed. + """ + vectorizer = TfidfVectorizer(ngram_range=(1, 3), min_df=1, max_df=0.95) + tfidf_matrix = vectorizer.fit_transform(documents) + matrix: list[list[float]] = cosine_similarity(tfidf_matrix).tolist() + return matrix + + +def find_similar_passages( + text1: str, text2: str, min_words: int = 5 +) -> list[tuple[str, str, float]]: + """Find similar sentence-level passages between two texts.""" + sentences1 = [s for s in sent_tokenize(text1) if len(s.split()) >= min_words] + sentences2 = [s for s in sent_tokenize(text2) if len(s.split()) >= min_words] + + if not sentences1 or not sentences2: + return [] + + all_sentences = sentences1 + sentences2 + preprocessed = [preprocess_text(s) for s in all_sentences] + + try: + vectorizer = TfidfVectorizer(ngram_range=(1, 2)) + tfidf_matrix = vectorizer.fit_transform(preprocessed) + except ValueError: + return [] + + n1 = len(sentences1) + similarities = [] + + for i, s1 in enumerate(sentences1): + for j, s2 in enumerate(sentences2): + sim = cosine_similarity( + tfidf_matrix[i : i + 1], tfidf_matrix[n1 + j : n1 + j + 1] + )[0][0] + if sim > PASSAGE_SIMILARITY_THRESHOLD: + similarities.append((s1, s2, sim)) + + return sorted(similarities, key=lambda x: x[2], reverse=True) diff --git a/linux_configuration/scripts/single_use/utils/sherlock.py b/linux_configuration/scripts/single_use/utils/sherlock.py new file mode 100755 index 0000000..0e9460d --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/sherlock.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Sherlock - simple text plagiarism detector using n-gram fingerprinting. + +Extracted from `install_plagiarism_tools.sh`'s fallback path, which used to +write this out via a `cat <<'SHERLOCKEOF'` heredoc when cloning the real +sherlock-py failed. Inline heredoc Python is invisible to ruff, mypy, pylint, +bandit and pytest, so it lives in its own module instead, matching +`download_nltk_data.py`'s extraction. + +The installer copies this file into `$SHERLOCK_DIR/sherlock.py` only when +`git clone` of the upstream tool fails. +""" + +from __future__ import annotations + +import argparse +import hashlib +from itertools import combinations +from pathlib import Path +import sys + +#: Minimum number of input files for a pairwise comparison to make sense. +MIN_FILES = 2 + + +def _out(line: str = "") -> None: + sys.stdout.write(line + "\n") + + +def tokenize(text: str) -> list[str]: + """Split text into lowercase alphanumeric words.""" + return [w.lower() for w in text.split() if w.isalnum()] + + +def get_ngrams(tokens: list[str], n: int = 3) -> list[tuple[str, ...]]: + """Generate n-grams from a token list.""" + return [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)] + + +def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set[str]: + """Create a document fingerprint from sampled n-gram hashes.""" + tokens = tokenize(text) + ngrams = get_ngrams(tokens, ngram_size) + + fingerprints = set() + for i, ng in enumerate(ngrams): + if i % sample_rate == 0: + h = hashlib.md5("".join(ng).encode(), usedforsecurity=False).hexdigest()[:8] + fingerprints.add(h) + + return fingerprints + + +def compare_documents(fp1: set[str], fp2: set[str]) -> float: + """Jaccard similarity between two fingerprints.""" + if not fp1 or not fp2: + return 0.0 + intersection = len(fp1 & fp2) + union = len(fp1 | fp2) + return intersection / union if union > 0 else 0.0 + + +def read_document(filepath: str) -> str: + """Read a document's raw text content.""" + with Path(filepath).open(encoding="utf-8", errors="ignore") as f: + return f.read() + + +def main() -> int: + """Fingerprint every input file and report pairwise similarity.""" + parser = argparse.ArgumentParser(description="Sherlock - Text Plagiarism Detector") + parser.add_argument("files", nargs="+", help="Files to compare") + parser.add_argument( + "--ngram", "-n", type=int, default=3, help="N-gram size (default: 3)" + ) + parser.add_argument( + "--threshold", "-t", type=float, default=0.1, help="Similarity threshold" + ) + + args = parser.parse_args() + + if len(args.files) < MIN_FILES: + _out("Need at least 2 files to compare") + return 1 + + docs = {} + for f in args.files: + if Path(f).exists(): + text = read_document(f) + docs[f] = fingerprint(text, args.ngram) + + _out("\nSherlock Plagiarism Analysis") + _out("=" * 50) + + files = list(docs.keys()) + for file1, file2 in combinations(files, 2): + sim = compare_documents(docs[file1], docs[file2]) + name1 = Path(file1).name + name2 = Path(file2).name + flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else "" + _out(f"{name1} <-> {name2}: {sim:.1%}{flag}") + + _out("=" * 50) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())