mirror of
https://github.com/kuhyx/testsAndMisc-archive.git
synced 2026-11-10 20:27:04 +01:00
install_plagiarism_tools.sh: split 534 -> 94 + 3 libs, with a real test harness
Wraps the four install stages into named functions, one per lib/plagiarism_*.sh sibling (matching install_exercism.sh's convention), and extracts the two embedded Python heredocs into real, lint-clean check_plagiarism.py/sherlock.py files -- check_plagiarism.py needed a second split into plagiarism_similarity.py to also clear the 250-line cap. Every function reaches 100% shell coverage via a PATH-shim test harness (fake python3/pip/git/java/wine); the two Python CLIs were separately verified against a real venv with actual nltk/scikit-learn. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
This commit is contained in:
parent
0186100a56
commit
e47c72329a
165
linux_configuration/scripts/single_use/utils/check_plagiarism.py
Executable file
165
linux_configuration/scripts/single_use/utils/check_plagiarism.py
Executable file
@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare documents for similarity using TF-IDF cosine similarity.
|
||||
|
||||
Extracted from `install_plagiarism_tools.sh`, which used to write this out via
|
||||
a `cat <<'PYEOF'` heredoc. Inline heredoc Python is invisible to ruff, mypy,
|
||||
pylint, bandit and pytest, so it lives in its own module instead, matching
|
||||
`download_nltk_data.py`'s extraction. The comparison engine (file reading,
|
||||
preprocessing, TF-IDF scoring) lives in `plagiarism_similarity.py`, split out
|
||||
to keep both files under the 250-line cap.
|
||||
|
||||
The installer copies this file into `$INSTALL_DIR/check_plagiarism.py` and
|
||||
wires the `plagcheck` wrapper to run it inside the venv.
|
||||
|
||||
Usage:
|
||||
python check_plagiarism.py file1.txt file2.txt [file3.txt ...]
|
||||
python check_plagiarism.py --dir /path/to/documents/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from plagiarism_similarity import (
|
||||
compute_similarity_matrix,
|
||||
find_similar_passages,
|
||||
preprocess_text,
|
||||
read_file,
|
||||
)
|
||||
|
||||
#: Minimum number of documents needed for a pairwise comparison to make sense.
|
||||
MIN_DOCUMENTS = 2
|
||||
|
||||
|
||||
def _out(line: str = "") -> None:
|
||||
sys.stdout.write(line + "\n")
|
||||
|
||||
|
||||
def _collect_files(args: argparse.Namespace) -> list[str]:
|
||||
files: list[str] = list(args.files)
|
||||
if args.dir:
|
||||
dir_path = Path(args.dir)
|
||||
for ext in ("*.txt", "*.pdf", "*.docx", "*.md", "*.tex"):
|
||||
files.extend(str(f) for f in dir_path.glob(ext))
|
||||
return files
|
||||
|
||||
|
||||
def _read_documents(files: list[str]) -> tuple[list[str], list[str]]:
|
||||
documents: list[str] = []
|
||||
filenames: list[str] = []
|
||||
for f in files:
|
||||
if not Path(f).exists():
|
||||
_out(f"Warning: {f} does not exist")
|
||||
continue
|
||||
text = read_file(f)
|
||||
if text.strip():
|
||||
documents.append(preprocess_text(text))
|
||||
filenames.append(Path(f).name)
|
||||
else:
|
||||
_out(f"Warning: {f} is empty or unreadable")
|
||||
return documents, filenames
|
||||
|
||||
|
||||
def _report_pairs(
|
||||
documents: list[str],
|
||||
filenames: list[str],
|
||||
sim_matrix: list[list[float]],
|
||||
threshold: float,
|
||||
) -> list[tuple[int, int, float, str]]:
|
||||
_out(f"{'Document Pair':<50} {'Similarity':>12}")
|
||||
_out("-" * 62)
|
||||
|
||||
suspicious_pairs: list[tuple[int, int, float, str]] = []
|
||||
for i in range(len(documents)):
|
||||
for j in range(i + 1, len(documents)):
|
||||
similarity = sim_matrix[i][j]
|
||||
pair_name = f"{filenames[i]} <-> {filenames[j]}"
|
||||
if similarity >= threshold:
|
||||
suspicious_pairs.append((i, j, similarity, pair_name))
|
||||
_out(f"{pair_name:<50} {similarity:>10.1%} ⚠️")
|
||||
else:
|
||||
_out(f"{pair_name:<50} {similarity:>10.1%}")
|
||||
|
||||
_out("-" * 62)
|
||||
return suspicious_pairs
|
||||
|
||||
|
||||
def _print_detailed_passages(
|
||||
files: list[str], suspicious_pairs: list[tuple[int, int, float, str]]
|
||||
) -> None:
|
||||
_out("\n" + "=" * 60)
|
||||
_out(" Detailed Similar Passages")
|
||||
_out("=" * 60)
|
||||
|
||||
for i, j, sim, pair_name in suspicious_pairs[:3]:
|
||||
_out(f"\n{pair_name} ({sim:.1%} similar):")
|
||||
_out("-" * 40)
|
||||
|
||||
raw_docs = [read_file(files[i]), read_file(files[j])]
|
||||
passages = find_similar_passages(raw_docs[0], raw_docs[1])
|
||||
|
||||
for s1, s2, psim in passages[:5]:
|
||||
_out(f'\n[{psim:.0%}] Document 1: "{s1[:100]}..."')
|
||||
_out(f' Document 2: "{s2[:100]}..."')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Compare every pair of input documents and report suspicious pairs."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Text Plagiarism Checker - Compare documents for similarity"
|
||||
)
|
||||
parser.add_argument("files", nargs="*", help="Files to compare")
|
||||
parser.add_argument("--dir", "-d", help="Directory containing documents to compare")
|
||||
parser.add_argument(
|
||||
"--threshold",
|
||||
"-t",
|
||||
type=float,
|
||||
default=0.3,
|
||||
help="Similarity threshold for flagging (0-1, default: 0.3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--detailed", "-v", action="store_true", help="Show detailed similar passages"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
files = _collect_files(args)
|
||||
if len(files) < MIN_DOCUMENTS:
|
||||
_out("Error: Need at least 2 files to compare")
|
||||
parser.print_help()
|
||||
return 1
|
||||
|
||||
_out(f"\n{'=' * 60}")
|
||||
_out(f" Plagiarism Check - Analyzing {len(files)} documents")
|
||||
_out(f"{'=' * 60}\n")
|
||||
|
||||
documents, filenames = _read_documents(files)
|
||||
if len(documents) < MIN_DOCUMENTS:
|
||||
_out("Error: Not enough valid documents to compare")
|
||||
return 1
|
||||
|
||||
_out("Computing document similarities...\n")
|
||||
sim_matrix = compute_similarity_matrix(documents)
|
||||
|
||||
suspicious_pairs = _report_pairs(documents, filenames, sim_matrix, args.threshold)
|
||||
|
||||
if suspicious_pairs:
|
||||
_out(
|
||||
f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed "
|
||||
f"{args.threshold:.0%} similarity threshold\n"
|
||||
)
|
||||
if args.detailed:
|
||||
_print_detailed_passages(files, suspicious_pairs)
|
||||
else:
|
||||
_out(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold")
|
||||
|
||||
_out("\n" + "=" * 60)
|
||||
_out(" Analysis complete")
|
||||
_out("=" * 60 + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -13,12 +13,6 @@ set -euo pipefail
|
||||
INSTALL_DIR="${HOME}/.local/share/plagiarism-tools"
|
||||
VENV_DIR="${HOME}/.local/share/plagiarism-venv"
|
||||
|
||||
echo "=============================================="
|
||||
echo " Open Source Plagiarism Detection Installer"
|
||||
echo " For Academic Text (Theses, Papers, etc.)"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
@ -29,506 +23,72 @@ success() { echo -e "${GREEN}✓ $1${NC}"; }
|
||||
warn() { echo -e "${YELLOW}⚠ $1${NC}"; }
|
||||
error() { echo -e "${RED}✗ $1${NC}"; }
|
||||
|
||||
# Create installation directory
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# 1. Python-based NLP Plagiarism Detection Environment
|
||||
# ------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=== 1. Installing Python NLP-based Plagiarism Tools ==="
|
||||
|
||||
# Check for Python 3
|
||||
if ! command -v python3 &>/dev/null; then
|
||||
error "Python 3 is required but not installed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create virtual environment
|
||||
if [ ! -d "$VENV_DIR" ]; then
|
||||
echo "Creating Python virtual environment..."
|
||||
python3 -m venv "$VENV_DIR"
|
||||
success "Virtual environment created at $VENV_DIR"
|
||||
else
|
||||
warn "Virtual environment already exists at $VENV_DIR"
|
||||
fi
|
||||
|
||||
# Activate and install packages
|
||||
# Created at runtime by `python -m venv` above, so there is nothing on disk
|
||||
# for the linter to follow at check time.
|
||||
# shellcheck disable=SC1091
|
||||
source "$VENV_DIR/bin/activate"
|
||||
|
||||
echo "Installing Python packages for text similarity detection..."
|
||||
pip install --upgrade pip
|
||||
|
||||
pip install --progress-bar on \
|
||||
scikit-learn \
|
||||
nltk \
|
||||
spacy \
|
||||
gensim \
|
||||
numpy \
|
||||
pandas \
|
||||
python-docx \
|
||||
PyPDF2 \
|
||||
beautifulsoup4 \
|
||||
lxml \
|
||||
textdistance \
|
||||
fuzzywuzzy \
|
||||
python-Levenshtein
|
||||
|
||||
success "Python NLP packages installed"
|
||||
|
||||
# Download NLTK data
|
||||
echo "Downloading NLTK data (stopwords, punkt tokenizer)..."
|
||||
python3 "$(dirname "$0")/download_nltk_data.py"
|
||||
success "NLTK data downloaded"
|
||||
|
||||
# Download spaCy English model (small)
|
||||
echo "Downloading spaCy English model..."
|
||||
python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm"
|
||||
success "spaCy model installed"
|
||||
|
||||
# Create a simple plagiarism checker script
|
||||
cat >"$INSTALL_DIR/check_plagiarism.py" <<'PYEOF'
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple Text Plagiarism Checker
|
||||
Compares documents using multiple similarity algorithms.
|
||||
|
||||
Usage:
|
||||
python check_plagiarism.py file1.txt file2.txt [file3.txt ...]
|
||||
python check_plagiarism.py --dir /path/to/documents/
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
import numpy as np
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
import nltk
|
||||
from nltk.corpus import stopwords
|
||||
from nltk.tokenize import word_tokenize, sent_tokenize
|
||||
|
||||
# Ensure NLTK data is available
|
||||
try:
|
||||
stopwords.words('english')
|
||||
except LookupError:
|
||||
nltk.download('stopwords', quiet=True)
|
||||
nltk.download('punkt', quiet=True)
|
||||
|
||||
|
||||
def read_file(filepath: str) -> str:
|
||||
"""Read text from various file formats."""
|
||||
path = Path(filepath)
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
if suffix == '.pdf':
|
||||
try:
|
||||
from PyPDF2 import PdfReader
|
||||
reader = PdfReader(filepath)
|
||||
return ' '.join(page.extract_text() or '' for page in reader.pages)
|
||||
except ImportError:
|
||||
print("Warning: PyPDF2 not installed, cannot read PDF files")
|
||||
return ""
|
||||
elif suffix == '.docx':
|
||||
try:
|
||||
from docx import Document
|
||||
doc = Document(filepath)
|
||||
return ' '.join(para.text for para in doc.paragraphs)
|
||||
except ImportError:
|
||||
print("Warning: python-docx not installed, cannot read DOCX files")
|
||||
return ""
|
||||
else:
|
||||
# Assume plain text
|
||||
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def preprocess_text(text: str) -> str:
|
||||
"""Clean and preprocess text for comparison."""
|
||||
# Lowercase
|
||||
text = text.lower()
|
||||
# Tokenize and remove stopwords
|
||||
try:
|
||||
stop_words = set(stopwords.words('english'))
|
||||
words = word_tokenize(text)
|
||||
words = [w for w in words if w.isalnum() and w not in stop_words]
|
||||
return ' '.join(words)
|
||||
except Exception:
|
||||
# Fallback: simple preprocessing
|
||||
return ' '.join(text.split())
|
||||
|
||||
|
||||
def compute_similarity_matrix(documents: List[str]) -> np.ndarray:
|
||||
"""Compute TF-IDF cosine similarity matrix."""
|
||||
vectorizer = TfidfVectorizer(
|
||||
ngram_range=(1, 3), # Use unigrams, bigrams, trigrams
|
||||
min_df=1,
|
||||
max_df=0.95
|
||||
)
|
||||
tfidf_matrix = vectorizer.fit_transform(documents)
|
||||
return cosine_similarity(tfidf_matrix)
|
||||
|
||||
|
||||
def find_similar_passages(text1: str, text2: str, min_words: int = 5) -> List[Tuple[str, str, float]]:
|
||||
"""Find similar sentence-level passages between two texts."""
|
||||
sentences1 = sent_tokenize(text1)
|
||||
sentences2 = sent_tokenize(text2)
|
||||
|
||||
if not sentences1 or not sentences2:
|
||||
return []
|
||||
|
||||
# Filter short sentences
|
||||
sentences1 = [s for s in sentences1 if len(s.split()) >= min_words]
|
||||
sentences2 = [s for s in sentences2 if len(s.split()) >= min_words]
|
||||
|
||||
if not sentences1 or not sentences2:
|
||||
return []
|
||||
|
||||
all_sentences = sentences1 + sentences2
|
||||
preprocessed = [preprocess_text(s) for s in all_sentences]
|
||||
|
||||
try:
|
||||
vectorizer = TfidfVectorizer(ngram_range=(1, 2))
|
||||
tfidf_matrix = vectorizer.fit_transform(preprocessed)
|
||||
except ValueError:
|
||||
return []
|
||||
|
||||
n1 = len(sentences1)
|
||||
similarities = []
|
||||
|
||||
for i, s1 in enumerate(sentences1):
|
||||
for j, s2 in enumerate(sentences2):
|
||||
sim = cosine_similarity(
|
||||
tfidf_matrix[i:i+1],
|
||||
tfidf_matrix[n1+j:n1+j+1]
|
||||
)[0][0]
|
||||
if sim > 0.5: # Threshold for suspicious similarity
|
||||
similarities.append((s1, s2, sim))
|
||||
|
||||
return sorted(similarities, key=lambda x: x[2], reverse=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Text Plagiarism Checker - Compare documents for similarity'
|
||||
)
|
||||
parser.add_argument('files', nargs='*', help='Files to compare')
|
||||
parser.add_argument('--dir', '-d', help='Directory containing documents to compare')
|
||||
parser.add_argument('--threshold', '-t', type=float, default=0.3,
|
||||
help='Similarity threshold for flagging (0-1, default: 0.3)')
|
||||
parser.add_argument('--detailed', '-v', action='store_true',
|
||||
help='Show detailed similar passages')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Collect files
|
||||
files = []
|
||||
if args.files:
|
||||
files.extend(args.files)
|
||||
if args.dir:
|
||||
dir_path = Path(args.dir)
|
||||
for ext in ['*.txt', '*.pdf', '*.docx', '*.md', '*.tex']:
|
||||
files.extend(str(f) for f in dir_path.glob(ext))
|
||||
|
||||
if len(files) < 2:
|
||||
print("Error: Need at least 2 files to compare")
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" Plagiarism Check - Analyzing {len(files)} documents")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Read and preprocess documents
|
||||
documents = []
|
||||
filenames = []
|
||||
for f in files:
|
||||
if os.path.exists(f):
|
||||
text = read_file(f)
|
||||
if text.strip():
|
||||
documents.append(preprocess_text(text))
|
||||
filenames.append(os.path.basename(f))
|
||||
else:
|
||||
print(f"Warning: {f} is empty or unreadable")
|
||||
else:
|
||||
print(f"Warning: {f} does not exist")
|
||||
|
||||
if len(documents) < 2:
|
||||
print("Error: Not enough valid documents to compare")
|
||||
sys.exit(1)
|
||||
|
||||
# Compute similarity
|
||||
print("Computing document similarities...\n")
|
||||
sim_matrix = compute_similarity_matrix(documents)
|
||||
|
||||
# Report results
|
||||
print(f"{'Document Pair':<50} {'Similarity':>12}")
|
||||
print("-" * 62)
|
||||
|
||||
suspicious_pairs = []
|
||||
for i in range(len(documents)):
|
||||
for j in range(i + 1, len(documents)):
|
||||
similarity = sim_matrix[i][j]
|
||||
pair_name = f"{filenames[i]} <-> {filenames[j]}"
|
||||
|
||||
if similarity >= args.threshold:
|
||||
suspicious_pairs.append((i, j, similarity, pair_name))
|
||||
print(f"{pair_name:<50} {similarity:>10.1%} ⚠️")
|
||||
else:
|
||||
print(f"{pair_name:<50} {similarity:>10.1%}")
|
||||
|
||||
print("-" * 62)
|
||||
|
||||
# Summary
|
||||
if suspicious_pairs:
|
||||
print(f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed {args.threshold:.0%} similarity threshold\n")
|
||||
|
||||
if args.detailed:
|
||||
print("\n" + "="*60)
|
||||
print(" Detailed Similar Passages")
|
||||
print("="*60)
|
||||
|
||||
for i, j, sim, pair_name in suspicious_pairs[:3]: # Limit to top 3
|
||||
print(f"\n{pair_name} ({sim:.1%} similar):")
|
||||
print("-" * 40)
|
||||
|
||||
raw_docs = [read_file(files[i]), read_file(files[j])]
|
||||
passages = find_similar_passages(raw_docs[0], raw_docs[1])
|
||||
|
||||
for s1, s2, psim in passages[:5]: # Top 5 passages
|
||||
print(f"\n[{psim:.0%}] Document 1: \"{s1[:100]}...\"")
|
||||
print(f" Document 2: \"{s2[:100]}...\"")
|
||||
else:
|
||||
print(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold")
|
||||
|
||||
print("\n" + "="*60)
|
||||
print(" Analysis complete")
|
||||
print("="*60 + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
PYEOF
|
||||
|
||||
chmod +x "$INSTALL_DIR/check_plagiarism.py"
|
||||
success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py"
|
||||
|
||||
# Create convenience wrapper
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
cat >"$HOME/.local/bin/plagcheck" <<WRAPEOF
|
||||
#!/usr/bin/env bash
|
||||
# Wrapper for plagiarism checker
|
||||
source "$VENV_DIR/bin/activate"
|
||||
python "$INSTALL_DIR/check_plagiarism.py" "\$@"
|
||||
WRAPEOF
|
||||
chmod +x "$HOME/.local/bin/plagcheck"
|
||||
success "Created 'plagcheck' command in ~/.local/bin/"
|
||||
|
||||
deactivate
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# 2. Sherlock for Text (Clone from GitHub)
|
||||
# ------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=== 2. Installing Sherlock Text Plagiarism Detector ==="
|
||||
|
||||
SHERLOCK_DIR="$INSTALL_DIR/sherlock"
|
||||
if [ ! -d "$SHERLOCK_DIR" ]; then
|
||||
# There are several Sherlock implementations; using a popular Python one
|
||||
if command -v git &>/dev/null; then
|
||||
# Clone a text-based similarity tool
|
||||
git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$SHERLOCK_DIR" 2>/dev/null || {
|
||||
warn "Could not clone sherlock-py, trying alternative..."
|
||||
# Alternative: Create a simple n-gram based sherlock
|
||||
mkdir -p "$SHERLOCK_DIR"
|
||||
cat >"$SHERLOCK_DIR/sherlock.py" <<'SHERLOCKEOF'
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sherlock - Simple text plagiarism detector using n-gram fingerprinting.
|
||||
Based on the original Sherlock algorithm.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def tokenize(text: str) -> list:
|
||||
"""Simple word tokenization."""
|
||||
return [w.lower() for w in text.split() if w.isalnum()]
|
||||
|
||||
|
||||
def get_ngrams(tokens: list, n: int = 3) -> list:
|
||||
"""Generate n-grams from token list."""
|
||||
return [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)]
|
||||
|
||||
|
||||
def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set:
|
||||
"""Create document fingerprint using sampled n-gram hashes."""
|
||||
tokens = tokenize(text)
|
||||
ngrams = get_ngrams(tokens, ngram_size)
|
||||
|
||||
fingerprints = set()
|
||||
for i, ng in enumerate(ngrams):
|
||||
if i % sample_rate == 0: # Sample every nth n-gram
|
||||
h = hashlib.md5(''.join(ng).encode()).hexdigest()[:8]
|
||||
fingerprints.add(h)
|
||||
|
||||
return fingerprints
|
||||
|
||||
|
||||
def compare_documents(fp1: set, fp2: set) -> float:
|
||||
"""Jaccard similarity between fingerprints."""
|
||||
if not fp1 or not fp2:
|
||||
return 0.0
|
||||
intersection = len(fp1 & fp2)
|
||||
union = len(fp1 | fp2)
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def read_document(filepath: str) -> str:
|
||||
"""Read document content."""
|
||||
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Sherlock - Text Plagiarism Detector')
|
||||
parser.add_argument('files', nargs='+', help='Files to compare')
|
||||
parser.add_argument('--ngram', '-n', type=int, default=3, help='N-gram size (default: 3)')
|
||||
parser.add_argument('--threshold', '-t', type=float, default=0.1, help='Similarity threshold')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(args.files) < 2:
|
||||
print("Need at least 2 files to compare")
|
||||
sys.exit(1)
|
||||
|
||||
# Read and fingerprint documents
|
||||
docs = {}
|
||||
for f in args.files:
|
||||
if os.path.exists(f):
|
||||
text = read_document(f)
|
||||
docs[f] = fingerprint(text, args.ngram)
|
||||
|
||||
print(f"\nSherlock Plagiarism Analysis")
|
||||
print("=" * 50)
|
||||
|
||||
# Compare all pairs
|
||||
files = list(docs.keys())
|
||||
for i in range(len(files)):
|
||||
for j in range(i + 1, len(files)):
|
||||
sim = compare_documents(docs[files[i]], docs[files[j]])
|
||||
name1 = os.path.basename(files[i])
|
||||
name2 = os.path.basename(files[j])
|
||||
flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else ""
|
||||
print(f"{name1} <-> {name2}: {sim:.1%}{flag}")
|
||||
|
||||
print("=" * 50)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
SHERLOCKEOF
|
||||
chmod +x "$SHERLOCK_DIR/sherlock.py"
|
||||
}
|
||||
success "Sherlock installed at $SHERLOCK_DIR"
|
||||
else
|
||||
warn "Git not available, skipping Sherlock installation"
|
||||
fi
|
||||
else
|
||||
warn "Sherlock already installed at $SHERLOCK_DIR"
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# 3. Ferret (Java-based) - Optional
|
||||
# ------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ==="
|
||||
|
||||
if command -v java &>/dev/null; then
|
||||
FERRET_DIR="$INSTALL_DIR/ferret"
|
||||
if [ ! -d "$FERRET_DIR" ]; then
|
||||
mkdir -p "$FERRET_DIR"
|
||||
echo "Ferret is a Java-based tool from University of Hertfordshire."
|
||||
echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/"
|
||||
echo "Place JAR file in: $FERRET_DIR"
|
||||
warn "Ferret requires manual download (academic license)"
|
||||
fi
|
||||
else
|
||||
warn "Java not installed, skipping Ferret"
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# 4. WCopyfind via Wine (Optional)
|
||||
# ------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ==="
|
||||
|
||||
if command -v wine &>/dev/null; then
|
||||
echo "Wine is available. WCopyfind can be run via Wine."
|
||||
echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/"
|
||||
echo "Run with: wine /path/to/WCopyfind.exe"
|
||||
warn "WCopyfind requires manual download"
|
||||
else
|
||||
echo "Wine not installed. To use WCopyfind:"
|
||||
echo " 1. Install wine: sudo apt install wine (or equivalent)"
|
||||
echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/"
|
||||
warn "WCopyfind skipped (Wine not available)"
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo " Installation Complete!"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
echo "Installed tools:"
|
||||
echo ""
|
||||
echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)"
|
||||
echo " Usage: plagcheck file1.txt file2.txt"
|
||||
echo " plagcheck --dir /path/to/documents/ --detailed"
|
||||
echo " Location: $INSTALL_DIR/check_plagiarism.py"
|
||||
echo ""
|
||||
echo "2. Sherlock (n-gram fingerprinting)"
|
||||
echo " Location: $SHERLOCK_DIR/sherlock.py"
|
||||
echo ""
|
||||
echo "3. Python virtual environment with NLP libraries:"
|
||||
echo " - scikit-learn (TF-IDF, cosine similarity)"
|
||||
echo " - NLTK (tokenization, stopwords)"
|
||||
echo " - spaCy (NLP processing)"
|
||||
echo " - gensim (document similarity)"
|
||||
echo " - textdistance, fuzzywuzzy (string matching)"
|
||||
echo " Activate with: source $VENV_DIR/bin/activate"
|
||||
echo ""
|
||||
echo "Quick Start:"
|
||||
echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed"
|
||||
echo " plagcheck --dir ./student_papers/ --threshold 0.4"
|
||||
echo ""
|
||||
echo "Note: Ensure ~/.local/bin is in your PATH:"
|
||||
# Printed for the user to paste into their shell rc: $HOME and $PATH must
|
||||
# stay literal rather than resolve to this machine's values.
|
||||
# shellcheck disable=SC2016
|
||||
echo ' export PATH="$HOME/.local/bin:$PATH"'
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
|
||||
# Add to PATH reminder
|
||||
if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then
|
||||
warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:"
|
||||
SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
|
||||
# shellcheck source=lib/plagiarism_python.sh
|
||||
source "$SCRIPT_DIR/lib/plagiarism_python.sh"
|
||||
# shellcheck source=lib/plagiarism_sherlock.sh
|
||||
source "$SCRIPT_DIR/lib/plagiarism_sherlock.sh"
|
||||
# shellcheck source=lib/plagiarism_optional.sh
|
||||
source "$SCRIPT_DIR/lib/plagiarism_optional.sh"
|
||||
|
||||
print_summary() {
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo " Installation Complete!"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
echo "Installed tools:"
|
||||
echo ""
|
||||
echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)"
|
||||
echo " Usage: plagcheck file1.txt file2.txt"
|
||||
echo " plagcheck --dir /path/to/documents/ --detailed"
|
||||
echo " Location: $INSTALL_DIR/check_plagiarism.py"
|
||||
echo ""
|
||||
echo "2. Sherlock (n-gram fingerprinting)"
|
||||
echo " Location: $INSTALL_DIR/sherlock/sherlock.py"
|
||||
echo ""
|
||||
echo "3. Python virtual environment with NLP libraries:"
|
||||
echo " - scikit-learn (TF-IDF, cosine similarity)"
|
||||
echo " - NLTK (tokenization, stopwords)"
|
||||
echo " - spaCy (NLP processing)"
|
||||
echo " - gensim (document similarity)"
|
||||
echo " - textdistance, fuzzywuzzy (string matching)"
|
||||
echo " Activate with: source $VENV_DIR/bin/activate"
|
||||
echo ""
|
||||
echo "Quick Start:"
|
||||
echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed"
|
||||
echo " plagcheck --dir ./student_papers/ --threshold 0.4"
|
||||
echo ""
|
||||
echo "Note: Ensure ~/.local/bin is in your PATH:"
|
||||
# Printed for the user to paste into their shell rc: $HOME and $PATH must
|
||||
# stay literal rather than resolve to this machine's values.
|
||||
# shellcheck disable=SC2016
|
||||
echo ' export PATH="$HOME/.local/bin:$PATH"'
|
||||
fi
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
|
||||
if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then
|
||||
warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:"
|
||||
# shellcheck disable=SC2016
|
||||
echo ' export PATH="$HOME/.local/bin:$PATH"'
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
echo "=============================================="
|
||||
echo " Open Source Plagiarism Detection Installer"
|
||||
echo " For Academic Text (Theses, Papers, etc.)"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
|
||||
install_python_nlp_tools "$SCRIPT_DIR"
|
||||
install_sherlock "$SCRIPT_DIR"
|
||||
check_ferret
|
||||
check_wcopyfind
|
||||
|
||||
print_summary
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
48
linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh
Executable file
48
linux_configuration/scripts/single_use/utils/lib/plagiarism_optional.sh
Executable file
@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# Stages 3-4: optional plagiarism tools that need a manual download.
|
||||
#
|
||||
# Sourced by install_plagiarism_tools.sh; split out to keep it under the
|
||||
# 250-line cap. Sourced rather than run, so it inherits the caller's strict
|
||||
# mode and the color/log helpers and INSTALL_DIR global defined above the
|
||||
# source line.
|
||||
|
||||
# Ferret (Java-based) - Optional
|
||||
check_ferret() {
|
||||
echo ""
|
||||
echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ==="
|
||||
|
||||
if ! command -v java &>/dev/null; then
|
||||
warn "Java not installed, skipping Ferret"
|
||||
return
|
||||
fi
|
||||
|
||||
local ferret_dir="$INSTALL_DIR/ferret"
|
||||
if [ -d "$ferret_dir" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
mkdir -p "$ferret_dir"
|
||||
echo "Ferret is a Java-based tool from University of Hertfordshire."
|
||||
echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/"
|
||||
echo "Place JAR file in: $ferret_dir"
|
||||
warn "Ferret requires manual download (academic license)"
|
||||
}
|
||||
|
||||
# WCopyfind via Wine (Optional)
|
||||
check_wcopyfind() {
|
||||
echo ""
|
||||
echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ==="
|
||||
|
||||
if command -v wine &>/dev/null; then
|
||||
echo "Wine is available. WCopyfind can be run via Wine."
|
||||
echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/"
|
||||
echo "Run with: wine /path/to/WCopyfind.exe"
|
||||
warn "WCopyfind requires manual download"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Wine not installed. To use WCopyfind:"
|
||||
echo " 1. Install wine: sudo apt install wine (or equivalent)"
|
||||
echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/"
|
||||
warn "WCopyfind skipped (Wine not available)"
|
||||
}
|
||||
80
linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh
Executable file
80
linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh
Executable file
@ -0,0 +1,80 @@
|
||||
#!/bin/bash
|
||||
# Stage 1: Python NLP-based plagiarism detection environment.
|
||||
#
|
||||
# Sourced by install_plagiarism_tools.sh; split out to keep it under the
|
||||
# 250-line cap. Sourced rather than run, so it inherits the caller's strict
|
||||
# mode and the color/log helpers and INSTALL_DIR/VENV_DIR globals defined
|
||||
# above the source line.
|
||||
|
||||
# Create the venv, pip-install the NLP stack, and install check_plagiarism.py
|
||||
# (plus the plagiarism_similarity.py module it imports) and its "plagcheck"
|
||||
# wrapper. $1 is the directory install_plagiarism_tools.sh lives in, so both
|
||||
# scripts can be copied from beside it.
|
||||
install_python_nlp_tools() {
|
||||
local script_dir="$1"
|
||||
|
||||
echo ""
|
||||
echo "=== 1. Installing Python NLP-based Plagiarism Tools ==="
|
||||
|
||||
if ! command -v python3 &>/dev/null; then
|
||||
error "Python 3 is required but not installed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$VENV_DIR" ]; then
|
||||
echo "Creating Python virtual environment..."
|
||||
python3 -m venv "$VENV_DIR"
|
||||
success "Virtual environment created at $VENV_DIR"
|
||||
else
|
||||
warn "Virtual environment already exists at $VENV_DIR"
|
||||
fi
|
||||
|
||||
# Created at runtime by `python -m venv` above, so there is nothing on disk
|
||||
# for the linter to follow at check time.
|
||||
# shellcheck disable=SC1091
|
||||
source "$VENV_DIR/bin/activate"
|
||||
|
||||
echo "Installing Python packages for text similarity detection..."
|
||||
pip install --upgrade pip
|
||||
|
||||
pip install --progress-bar on \
|
||||
scikit-learn \
|
||||
nltk \
|
||||
spacy \
|
||||
gensim \
|
||||
numpy \
|
||||
pandas \
|
||||
python-docx \
|
||||
PyPDF2 \
|
||||
beautifulsoup4 \
|
||||
lxml \
|
||||
textdistance \
|
||||
fuzzywuzzy \
|
||||
python-Levenshtein
|
||||
|
||||
success "Python NLP packages installed"
|
||||
|
||||
echo "Downloading NLTK data (stopwords, punkt tokenizer)..."
|
||||
python3 "$script_dir/download_nltk_data.py"
|
||||
success "NLTK data downloaded"
|
||||
|
||||
echo "Downloading spaCy English model..."
|
||||
python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm"
|
||||
success "spaCy model installed"
|
||||
|
||||
install -m 755 "$script_dir/check_plagiarism.py" "$INSTALL_DIR/check_plagiarism.py"
|
||||
install -m 644 "$script_dir/plagiarism_similarity.py" "$INSTALL_DIR/plagiarism_similarity.py"
|
||||
success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py"
|
||||
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
cat >"$HOME/.local/bin/plagcheck" <<WRAPEOF
|
||||
#!/usr/bin/env bash
|
||||
# Wrapper for plagiarism checker
|
||||
source "$VENV_DIR/bin/activate"
|
||||
python "$INSTALL_DIR/check_plagiarism.py" "\$@"
|
||||
WRAPEOF
|
||||
chmod +x "$HOME/.local/bin/plagcheck"
|
||||
success "Created 'plagcheck' command in ~/.local/bin/"
|
||||
|
||||
deactivate
|
||||
}
|
||||
40
linux_configuration/scripts/single_use/utils/lib/plagiarism_sherlock.sh
Executable file
40
linux_configuration/scripts/single_use/utils/lib/plagiarism_sherlock.sh
Executable file
@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# Stage 2: Sherlock text plagiarism detector.
|
||||
#
|
||||
# Sourced by install_plagiarism_tools.sh; split out to keep it under the
|
||||
# 250-line cap. Sourced rather than run, so it inherits the caller's strict
|
||||
# mode and the color/log helpers and INSTALL_DIR global defined above the
|
||||
# source line.
|
||||
|
||||
# Clone the upstream sherlock-py tool, falling back to the bundled
|
||||
# sherlock.py (n-gram fingerprinting) when git or the clone is unavailable.
|
||||
# $1 is the directory install_plagiarism_tools.sh lives in, so the fallback
|
||||
# sherlock.py can be copied from beside it.
|
||||
install_sherlock() {
|
||||
local script_dir="$1"
|
||||
|
||||
echo ""
|
||||
echo "=== 2. Installing Sherlock Text Plagiarism Detector ==="
|
||||
|
||||
local sherlock_dir="$INSTALL_DIR/sherlock"
|
||||
if [ -d "$sherlock_dir" ]; then
|
||||
warn "Sherlock already installed at $sherlock_dir"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! command -v git &>/dev/null; then
|
||||
warn "Git not available, skipping Sherlock installation"
|
||||
return
|
||||
fi
|
||||
|
||||
# There are several Sherlock implementations; using a popular Python one
|
||||
if git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$sherlock_dir" 2>/dev/null; then
|
||||
success "Sherlock installed at $sherlock_dir"
|
||||
return
|
||||
fi
|
||||
|
||||
warn "Could not clone sherlock-py, trying alternative..."
|
||||
mkdir -p "$sherlock_dir"
|
||||
install -m 755 "$script_dir/sherlock.py" "$sherlock_dir/sherlock.py"
|
||||
success "Sherlock installed at $sherlock_dir"
|
||||
}
|
||||
130
linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh
Executable file
130
linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh
Executable file
@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/tests/plagiarism_harness.sh — fake python3/pip/git/java/wine behind the
|
||||
# install_plagiarism_tools.sh split.
|
||||
#
|
||||
# Sourced, not executed. Every external tool the installer calls is a shim on
|
||||
# PATH that records its invocation into $DEV and does the minimum real
|
||||
# filesystem work the calling code depends on (e.g. `python3 -m venv` must
|
||||
# leave a `bin/activate` a real `source` can run). INSTALL_DIR, VENV_DIR and
|
||||
# HOME all point into a throwaway tmpdir so nothing touches the real machine.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
UTILS_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
_t_pass() {
|
||||
PASS=$((PASS + 1))
|
||||
printf ' OK: %s\n' "$1"
|
||||
}
|
||||
|
||||
_t_fail() {
|
||||
FAIL=$((FAIL + 1))
|
||||
printf ' FAIL: %s\n' "$1"
|
||||
}
|
||||
|
||||
_t_eq() {
|
||||
local want="$1" got="$2" what="$3"
|
||||
if [[ "$got" == "$want" ]]; then
|
||||
_t_pass "$what"
|
||||
else
|
||||
_t_fail "$what (want '${want}', got '${got}')"
|
||||
fi
|
||||
}
|
||||
|
||||
TEST_TMPDIR="$(mktemp -d)"
|
||||
trap 'rm -rf "${TEST_TMPDIR}"' EXIT
|
||||
|
||||
readonly DEV="${TEST_TMPDIR}/device"
|
||||
readonly FAKE_BIN="${TEST_TMPDIR}/fake_bin"
|
||||
mkdir -p "${DEV}" "${FAKE_BIN}"
|
||||
|
||||
# --- fake external tools ----------------------------------------------------
|
||||
|
||||
cat >"${FAKE_BIN}/python3" <<'PYSHIM'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
DEV="${PLAGIARISM_TEST_DEV}"
|
||||
printf '%s\n' "python3 $*" >>"${DEV}/calls"
|
||||
if [[ "$1" == "-m" && "$2" == "venv" ]]; then
|
||||
venv_dir="$3"
|
||||
mkdir -p "${venv_dir}/bin"
|
||||
cat >"${venv_dir}/bin/activate" <<'ACTEOF'
|
||||
deactivate() { :; }
|
||||
pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; }
|
||||
ACTEOF
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$1" == "-m" && "$2" == "spacy" ]]; then
|
||||
[[ -f "${DEV}/fail_spacy" ]] && exit 1
|
||||
exit 0
|
||||
fi
|
||||
# python3 <script.py> [args...] — record which script ran and succeed.
|
||||
printf '%s\n' "ran:$1" >>"${DEV}/scripts_run"
|
||||
[[ -f "${DEV}/fail_$(basename "$1" .py)" ]] && exit 1
|
||||
exit 0
|
||||
PYSHIM
|
||||
chmod +x "${FAKE_BIN}/python3"
|
||||
|
||||
cat >"${FAKE_BIN}/git" <<'GITSHIM'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
DEV="${PLAGIARISM_TEST_DEV}"
|
||||
printf '%s\n' "git $*" >>"${DEV}/calls"
|
||||
if [[ "$1" == "clone" ]]; then
|
||||
[[ -f "${DEV}/fail_clone" ]] && exit 1
|
||||
dest="${*: -1}"
|
||||
mkdir -p "${dest}"
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
GITSHIM
|
||||
chmod +x "${FAKE_BIN}/git"
|
||||
|
||||
cat >"${FAKE_BIN}/java" <<'JAVASHIM'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
JAVASHIM
|
||||
chmod +x "${FAKE_BIN}/java"
|
||||
|
||||
cat >"${FAKE_BIN}/wine" <<'WINESHIM'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
WINESHIM
|
||||
chmod +x "${FAKE_BIN}/wine"
|
||||
|
||||
export PLAGIARISM_TEST_DEV="${DEV}"
|
||||
export PATH="${FAKE_BIN}:${PATH}"
|
||||
|
||||
# --- subject under test -----------------------------------------------------
|
||||
|
||||
HOME="${TEST_TMPDIR}/home"
|
||||
INSTALL_DIR="${TEST_TMPDIR}/install"
|
||||
VENV_DIR="${TEST_TMPDIR}/venv"
|
||||
mkdir -p "${HOME}" "${INSTALL_DIR}"
|
||||
|
||||
success() { :; }
|
||||
warn() { printf '%s\n' "$*" >>"${DEV}/warnings"; }
|
||||
error() { printf '%s\n' "$*" >>"${DEV}/errors"; }
|
||||
|
||||
# shellcheck source=../plagiarism_python.sh
|
||||
. "${UTILS_DIR}/lib/plagiarism_python.sh"
|
||||
# shellcheck source=../plagiarism_sherlock.sh
|
||||
. "${UTILS_DIR}/lib/plagiarism_sherlock.sh"
|
||||
# shellcheck source=../plagiarism_optional.sh
|
||||
. "${UTILS_DIR}/lib/plagiarism_optional.sh"
|
||||
|
||||
# reset_state — wipe INSTALL_DIR/VENV_DIR/DEV records between test groups so
|
||||
# each group starts from "nothing installed yet".
|
||||
reset_state() {
|
||||
rm -rf "${INSTALL_DIR}" "${VENV_DIR}" "${DEV:?}/calls" "${DEV}/scripts_run" \
|
||||
"${DEV}/warnings" "${DEV}/errors" 2>/dev/null || true
|
||||
mkdir -p "${INSTALL_DIR}"
|
||||
: >"${DEV}/calls"
|
||||
: >"${DEV}/scripts_run"
|
||||
: >"${DEV}/warnings"
|
||||
: >"${DEV}/errors"
|
||||
}
|
||||
reset_state
|
||||
@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end test for install_plagiarism_tools.sh's main()/print_summary(),
|
||||
# which stay in the entry script rather than a lib. Runs the real script as a
|
||||
# subprocess against the same fake python3/pip/git/java/wine as the lib tests,
|
||||
# with HOME pointed at a throwaway tmpdir so INSTALL_DIR/VENV_DIR (derived
|
||||
# from $HOME inside the script) land there too.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
readonly ENTRY_SCRIPT="${UTILS_DIR}/install_plagiarism_tools.sh"
|
||||
|
||||
# The entry script derives INSTALL_DIR/VENV_DIR from $HOME itself (a plain
|
||||
# assignment near its top), independently of the lib tests' INSTALL_DIR/
|
||||
# VENV_DIR harness variables. Running it as a real subprocess against the
|
||||
# harness's $HOME means its paths land under $HOME, not under the harness's
|
||||
# separate INSTALL_DIR/VENV_DIR -- so assertions below re-derive the same
|
||||
# paths the script itself computes, rather than reusing the harness ones.
|
||||
readonly ENTRY_INSTALL_DIR="${HOME}/.local/share/plagiarism-tools"
|
||||
readonly ENTRY_VENV_DIR="${HOME}/.local/share/plagiarism-venv"
|
||||
|
||||
# --- full run, PATH already in ~/.local/bin -------------------------------
|
||||
|
||||
mkdir -p "${HOME}/.local/bin"
|
||||
OUT_FULL="${TEST_TMPDIR}/out_full.log"
|
||||
PATH="${FAKE_BIN}:${HOME}/.local/bin:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_FULL}" 2>&1
|
||||
_status=$?
|
||||
|
||||
if [[ "${_status}" -eq 0 ]]; then
|
||||
_t_pass "install_plagiarism_tools.sh exits zero on a full successful run"
|
||||
else
|
||||
_t_fail "install_plagiarism_tools.sh should exit zero (log: $(cat "${OUT_FULL}"))"
|
||||
fi
|
||||
|
||||
if grep -q "Open Source Plagiarism Detection Installer" "${OUT_FULL}"; then
|
||||
_t_pass "main prints the installer banner"
|
||||
else
|
||||
_t_fail "main should print the installer banner"
|
||||
fi
|
||||
|
||||
if grep -q "=== 1. Installing Python NLP-based Plagiarism Tools ===" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 1 (Python NLP tools)"
|
||||
else
|
||||
_t_fail "main should run stage 1"
|
||||
fi
|
||||
|
||||
if grep -q "=== 2. Installing Sherlock Text Plagiarism Detector ===" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 2 (Sherlock)"
|
||||
else
|
||||
_t_fail "main should run stage 2"
|
||||
fi
|
||||
|
||||
if grep -q "=== 3. Checking for Ferret" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 3 (Ferret)"
|
||||
else
|
||||
_t_fail "main should run stage 3"
|
||||
fi
|
||||
|
||||
if grep -q "=== 4. WCopyfind Information" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 4 (WCopyfind)"
|
||||
else
|
||||
_t_fail "main should run stage 4"
|
||||
fi
|
||||
|
||||
if grep -q "Installation Complete!" "${OUT_FULL}"; then
|
||||
_t_pass "main prints the summary"
|
||||
else
|
||||
_t_fail "main should print the summary"
|
||||
fi
|
||||
|
||||
if grep -q "Location: ${ENTRY_INSTALL_DIR}/check_plagiarism.py" "${OUT_FULL}"; then
|
||||
_t_pass "the summary reports the checker's installed location"
|
||||
else
|
||||
_t_fail "the summary should report the checker's installed location"
|
||||
fi
|
||||
|
||||
if grep -q "Location: ${ENTRY_INSTALL_DIR}/sherlock/sherlock.py" "${OUT_FULL}"; then
|
||||
_t_pass "the summary reports sherlock's installed location"
|
||||
else
|
||||
_t_fail "the summary should report sherlock's installed location"
|
||||
fi
|
||||
|
||||
if grep -q "Activate with: source ${ENTRY_VENV_DIR}/bin/activate" "${OUT_FULL}"; then
|
||||
_t_pass "the summary reports the venv activation command"
|
||||
else
|
||||
_t_fail "the summary should report the venv activation command"
|
||||
fi
|
||||
|
||||
# ~/.local/bin was already on PATH for this run, so the "add it to your PATH"
|
||||
# nudge must not repeat in the summary.
|
||||
if grep -q "Add ~/.local/bin to your PATH by adding this" "${OUT_FULL}"; then
|
||||
_t_fail "the summary should not repeat the PATH nudge when ~/.local/bin is already on PATH"
|
||||
else
|
||||
_t_pass "the summary skips the PATH nudge when ~/.local/bin is already on PATH"
|
||||
fi
|
||||
|
||||
# --- ~/.local/bin missing from PATH: the summary nudges once --------------
|
||||
|
||||
reset_state
|
||||
rm -rf "${HOME}/.local"
|
||||
OUT_NOPATH="${TEST_TMPDIR}/out_nopath.log"
|
||||
PATH="${FAKE_BIN}:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_NOPATH}" 2>&1
|
||||
|
||||
if grep -qc "Add ~/.local/bin to your PATH by adding this" "${OUT_NOPATH}"; then
|
||||
_t_pass "the summary nudges to add ~/.local/bin to PATH when it is missing"
|
||||
else
|
||||
_t_fail "the summary should nudge to add ~/.local/bin to PATH when it is missing"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for lib/plagiarism_optional.sh (Ferret/WCopyfind availability
|
||||
# checks) against the fake java/wine in plagiarism_harness.sh.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
_no_tool_path() {
|
||||
local skip="$1"
|
||||
local dir="${TEST_TMPDIR}/no_${skip}_bin"
|
||||
mkdir -p "${dir}"
|
||||
local tool
|
||||
for tool in python3 git java wine cat chmod mkdir; do
|
||||
[[ "${tool}" == "${skip}" ]] && continue
|
||||
[[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${dir}/${tool}"
|
||||
done
|
||||
printf '%s' "${dir}"
|
||||
}
|
||||
|
||||
# --- check_ferret: java available -----------------------------------------
|
||||
|
||||
check_ferret >/tmp/plagiarism_optional_test.log 2>&1
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/ferret" ]]; then
|
||||
_t_pass "check_ferret creates the ferret directory when java is available"
|
||||
else
|
||||
_t_fail "check_ferret should create ${INSTALL_DIR}/ferret when java is available"
|
||||
fi
|
||||
|
||||
if grep -q "requires manual download" "${DEV}/warnings"; then
|
||||
_t_pass "check_ferret explains that Ferret needs a manual download"
|
||||
else
|
||||
_t_fail "check_ferret should explain that Ferret needs a manual download"
|
||||
fi
|
||||
|
||||
# check_ferret creates the directory as a marker; a second run must not
|
||||
# re-print the manual-download instructions once it already exists.
|
||||
: >"${DEV}/warnings"
|
||||
check_ferret >/tmp/plagiarism_optional_test.log 2>&1
|
||||
if grep -q "requires manual download" "${DEV}/warnings"; then
|
||||
_t_fail "check_ferret should not repeat instructions once the ferret dir already exists"
|
||||
else
|
||||
_t_pass "check_ferret is a no-op once the ferret directory already exists"
|
||||
fi
|
||||
|
||||
# --- check_ferret: java unavailable ---------------------------------------
|
||||
|
||||
reset_state
|
||||
no_java_path="$(_no_tool_path java)"
|
||||
PATH="${no_java_path}" check_ferret >/dev/null 2>&1
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/ferret" ]]; then
|
||||
_t_fail "check_ferret should not create the ferret directory without java"
|
||||
else
|
||||
_t_pass "check_ferret skips Ferret entirely when java is unavailable"
|
||||
fi
|
||||
|
||||
if grep -q "Java not installed" "${DEV}/warnings"; then
|
||||
_t_pass "check_ferret warns when java is unavailable"
|
||||
else
|
||||
_t_fail "check_ferret should warn when java is unavailable"
|
||||
fi
|
||||
|
||||
# --- check_wcopyfind: wine available ---------------------------------------
|
||||
|
||||
: >"${DEV}/warnings"
|
||||
check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1
|
||||
|
||||
if grep -q "Wine is available" /tmp/plagiarism_optional_test.log; then
|
||||
_t_pass "check_wcopyfind reports Wine as available"
|
||||
else
|
||||
_t_fail "check_wcopyfind should report Wine as available"
|
||||
fi
|
||||
|
||||
if grep -q "requires manual download" "${DEV}/warnings"; then
|
||||
_t_pass "check_wcopyfind warns that WCopyfind needs a manual download"
|
||||
else
|
||||
_t_fail "check_wcopyfind should warn that WCopyfind needs a manual download"
|
||||
fi
|
||||
|
||||
# --- check_wcopyfind: wine unavailable --------------------------------------
|
||||
|
||||
: >"${DEV}/warnings"
|
||||
no_wine_path="$(_no_tool_path wine)"
|
||||
PATH="${no_wine_path}" check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1
|
||||
|
||||
if grep -q "Wine not installed" /tmp/plagiarism_optional_test.log; then
|
||||
_t_pass "check_wcopyfind reports Wine as unavailable"
|
||||
else
|
||||
_t_fail "check_wcopyfind should report Wine as unavailable"
|
||||
fi
|
||||
|
||||
if grep -q "skipped (Wine not available)" "${DEV}/warnings"; then
|
||||
_t_pass "check_wcopyfind warns that WCopyfind was skipped"
|
||||
else
|
||||
_t_fail "check_wcopyfind should warn that WCopyfind was skipped"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
132
linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh
Executable file
132
linux_configuration/scripts/single_use/utils/lib/tests/test_plagiarism_python.sh
Executable file
@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for lib/plagiarism_python.sh against the fake python3/pip in
|
||||
# plagiarism_harness.sh. No real network or venv is needed.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
# --- happy path ---------------------------------------------------------
|
||||
|
||||
install_python_nlp_tools "${UTILS_DIR}" >/tmp/plagiarism_python_test.log 2>&1
|
||||
_status=$?
|
||||
if [[ "${_status}" -eq 0 ]]; then
|
||||
_t_pass "install_python_nlp_tools succeeds against a working toolchain"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should succeed (log: $(cat /tmp/plagiarism_python_test.log))"
|
||||
fi
|
||||
|
||||
if [[ -d "${VENV_DIR}" ]]; then
|
||||
_t_pass "install_python_nlp_tools creates the virtual environment"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should create ${VENV_DIR}"
|
||||
fi
|
||||
|
||||
if grep -q '^python3 -m venv' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools invokes python3 -m venv"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should invoke python3 -m venv"
|
||||
fi
|
||||
|
||||
if grep -q 'pip install --upgrade pip' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools upgrades pip"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should upgrade pip"
|
||||
fi
|
||||
|
||||
if grep -q 'scikit-learn' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools installs the NLP package set"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should install scikit-learn and friends"
|
||||
fi
|
||||
|
||||
if grep -q 'ran:.*download_nltk_data.py' "${DEV}/scripts_run"; then
|
||||
_t_pass "install_python_nlp_tools runs download_nltk_data.py"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should run download_nltk_data.py"
|
||||
fi
|
||||
|
||||
if grep -q '^python3 -m spacy download' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools downloads the spaCy model"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should download the spaCy model"
|
||||
fi
|
||||
|
||||
if [[ -x "${INSTALL_DIR}/check_plagiarism.py" ]]; then
|
||||
_t_pass "install_python_nlp_tools installs check_plagiarism.py, executable"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should install an executable check_plagiarism.py"
|
||||
fi
|
||||
|
||||
if [[ -f "${INSTALL_DIR}/plagiarism_similarity.py" ]]; then
|
||||
_t_pass "install_python_nlp_tools installs plagiarism_similarity.py alongside it"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should install plagiarism_similarity.py -- \
|
||||
check_plagiarism.py imports it, so a missing copy breaks at runtime, not install time"
|
||||
fi
|
||||
|
||||
if [[ -x "${HOME}/.local/bin/plagcheck" ]]; then
|
||||
_t_pass "install_python_nlp_tools creates an executable plagcheck wrapper"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should create an executable plagcheck wrapper"
|
||||
fi
|
||||
|
||||
if grep -q "${VENV_DIR}/bin/activate" "${HOME}/.local/bin/plagcheck"; then
|
||||
_t_pass "plagcheck wrapper activates the venv it was installed against"
|
||||
else
|
||||
_t_fail "plagcheck wrapper should reference ${VENV_DIR}/bin/activate"
|
||||
fi
|
||||
|
||||
if grep -q "${INSTALL_DIR}/check_plagiarism.py" "${HOME}/.local/bin/plagcheck"; then
|
||||
_t_pass "plagcheck wrapper points at the installed checker script"
|
||||
else
|
||||
_t_fail "plagcheck wrapper should point at ${INSTALL_DIR}/check_plagiarism.py"
|
||||
fi
|
||||
|
||||
# --- venv already exists: warn, don't recreate --------------------------
|
||||
|
||||
reset_state
|
||||
mkdir -p "${VENV_DIR}/bin"
|
||||
cat >"${VENV_DIR}/bin/activate" <<'ACTEOF'
|
||||
deactivate() { :; }
|
||||
pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; }
|
||||
ACTEOF
|
||||
: >"${DEV}/calls"
|
||||
install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1
|
||||
if grep -q '^python3 -m venv' "${DEV}/calls"; then
|
||||
_t_fail "install_python_nlp_tools should not recreate an existing venv"
|
||||
else
|
||||
_t_pass "install_python_nlp_tools skips venv creation when one already exists"
|
||||
fi
|
||||
|
||||
# --- python3 missing: fatal, not silently skipped ------------------------
|
||||
|
||||
reset_state
|
||||
# A bare `rm -f "${FAKE_BIN}/python3"` does not hide python3 from
|
||||
# `command -v`: pyenv's shim is still reachable further down the real PATH.
|
||||
# Instead point PATH at a directory that shadows python3 with nothing before
|
||||
# falling through to the rest of the fakes, so `command -v python3` fails.
|
||||
NO_PYTHON_BIN="${TEST_TMPDIR}/no_python_bin"
|
||||
mkdir -p "${NO_PYTHON_BIN}"
|
||||
for tool in git java wine cat chmod mkdir; do
|
||||
[[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_PYTHON_BIN}/${tool}"
|
||||
done
|
||||
# install_python_nlp_tools calls `exit` (not `return`) on a missing python3,
|
||||
# by design -- a partial install with no interpreter must abort the whole
|
||||
# installer, not just this stage. Run it in a subshell so that exit only
|
||||
# ends the subshell, not this test script.
|
||||
if (PATH="${NO_PYTHON_BIN}" install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1); then
|
||||
_t_fail "install_python_nlp_tools should exit non-zero without python3"
|
||||
else
|
||||
_t_pass "install_python_nlp_tools exits non-zero when python3 is missing"
|
||||
fi
|
||||
if grep -q "Python 3 is required" "${DEV}/errors"; then
|
||||
_t_pass "install_python_nlp_tools reports the missing-python3 error"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should report the missing-python3 error"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for lib/plagiarism_sherlock.sh against the fake git in
|
||||
# plagiarism_harness.sh. No real network is needed.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
# --- upstream clone succeeds ---------------------------------------------
|
||||
|
||||
install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if grep -q '^git clone' "${DEV}/calls"; then
|
||||
_t_pass "install_sherlock clones the upstream sherlock-py tool"
|
||||
else
|
||||
_t_fail "install_sherlock should attempt a git clone"
|
||||
fi
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/sherlock" ]]; then
|
||||
_t_pass "install_sherlock creates the sherlock directory on a successful clone"
|
||||
else
|
||||
_t_fail "install_sherlock should create ${INSTALL_DIR}/sherlock"
|
||||
fi
|
||||
|
||||
if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then
|
||||
_t_fail "a successful clone should not also install the bundled fallback sherlock.py"
|
||||
else
|
||||
_t_pass "a successful clone does not install the bundled fallback sherlock.py"
|
||||
fi
|
||||
|
||||
# --- already installed: no-op, warns -------------------------------------
|
||||
|
||||
: >"${DEV}/calls"
|
||||
install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if grep -q '^git clone' "${DEV}/calls"; then
|
||||
_t_fail "install_sherlock should not re-clone when already installed"
|
||||
else
|
||||
_t_pass "install_sherlock skips reinstalling when the directory already exists"
|
||||
fi
|
||||
|
||||
if grep -q "already installed" "${DEV}/warnings"; then
|
||||
_t_pass "install_sherlock warns that sherlock is already installed"
|
||||
else
|
||||
_t_fail "install_sherlock should warn that sherlock is already installed"
|
||||
fi
|
||||
|
||||
# --- clone fails: falls back to the bundled sherlock.py -------------------
|
||||
|
||||
reset_state
|
||||
touch "${DEV}/fail_clone"
|
||||
install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then
|
||||
_t_pass "install_sherlock falls back to the bundled sherlock.py when the clone fails"
|
||||
else
|
||||
_t_fail "install_sherlock should fall back to the bundled sherlock.py when the clone fails"
|
||||
fi
|
||||
|
||||
if [[ -x "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then
|
||||
_t_pass "the bundled fallback sherlock.py is installed executable"
|
||||
else
|
||||
_t_fail "the bundled fallback sherlock.py should be installed executable"
|
||||
fi
|
||||
|
||||
if grep -q "Could not clone sherlock-py" "${DEV}/warnings"; then
|
||||
_t_pass "install_sherlock warns when the clone fails, before falling back"
|
||||
else
|
||||
_t_fail "install_sherlock should warn when the clone fails"
|
||||
fi
|
||||
|
||||
# The installed fallback must be byte-identical to the source file: it is the
|
||||
# one artifact the git-unavailable path can serve, so a stale copy would ship
|
||||
# broken plagiarism detection silently.
|
||||
if diff -q "${UTILS_DIR}/sherlock.py" "${INSTALL_DIR}/sherlock/sherlock.py" >/dev/null; then
|
||||
_t_pass "the installed fallback sherlock.py matches the source file byte-for-byte"
|
||||
else
|
||||
_t_fail "the installed fallback sherlock.py should match ${UTILS_DIR}/sherlock.py"
|
||||
fi
|
||||
|
||||
# --- git not available: warns, does not attempt to install ----------------
|
||||
|
||||
reset_state
|
||||
NO_GIT_BIN="${TEST_TMPDIR}/no_git_bin"
|
||||
mkdir -p "${NO_GIT_BIN}"
|
||||
for tool in python3 java wine cat chmod mkdir; do
|
||||
[[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_GIT_BIN}/${tool}"
|
||||
done
|
||||
PATH="${NO_GIT_BIN}" install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/sherlock" ]]; then
|
||||
_t_fail "install_sherlock should not create the sherlock directory without git"
|
||||
else
|
||||
_t_pass "install_sherlock skips installation entirely when git is unavailable"
|
||||
fi
|
||||
|
||||
if grep -q "Git not available" "${DEV}/warnings"; then
|
||||
_t_pass "install_sherlock warns when git is unavailable"
|
||||
else
|
||||
_t_fail "install_sherlock should warn when git is unavailable"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
141
linux_configuration/scripts/single_use/utils/plagiarism_similarity.py
Executable file
141
linux_configuration/scripts/single_use/utils/plagiarism_similarity.py
Executable file
@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Document reading and TF-IDF similarity scoring for check_plagiarism.py.
|
||||
|
||||
Split out of `check_plagiarism.py` to keep it under the 250-line cap: this
|
||||
module holds the comparison engine (file reading, text preprocessing, and the
|
||||
TF-IDF/cosine-similarity computations), `check_plagiarism.py` keeps the CLI
|
||||
and reporting.
|
||||
|
||||
Its third-party imports (numpy, sklearn, nltk) are the plagiarism toolchain's
|
||||
dependencies, not this repo's, so they are absent from `meta/requirements.txt`
|
||||
on purpose. Importing them dynamically (rather than with a static `import
|
||||
nltk`) keeps this module importable -- and lintable -- in an environment
|
||||
that doesn't have them installed, matching `download_nltk_data.py`'s
|
||||
extraction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
#: Cosine similarity above this, for a single sentence pair, is worth reporting
|
||||
#: as a suspicious passage in --detailed mode.
|
||||
PASSAGE_SIMILARITY_THRESHOLD = 0.5
|
||||
|
||||
nltk = importlib.import_module("nltk")
|
||||
stopwords = importlib.import_module("nltk.corpus").stopwords
|
||||
nltk_tokenize = importlib.import_module("nltk.tokenize")
|
||||
sent_tokenize = nltk_tokenize.sent_tokenize
|
||||
word_tokenize = nltk_tokenize.word_tokenize
|
||||
TfidfVectorizer = importlib.import_module(
|
||||
"sklearn.feature_extraction.text"
|
||||
).TfidfVectorizer
|
||||
cosine_similarity = importlib.import_module(
|
||||
"sklearn.metrics.pairwise"
|
||||
).cosine_similarity
|
||||
|
||||
try:
|
||||
stopwords.words("english")
|
||||
except LookupError:
|
||||
nltk.download("stopwords", quiet=True)
|
||||
nltk.download("punkt", quiet=True)
|
||||
|
||||
|
||||
def _out(line: str = "") -> None:
|
||||
sys.stdout.write(line + "\n")
|
||||
|
||||
|
||||
def _read_pdf(filepath: str) -> str:
|
||||
try:
|
||||
pypdf2 = importlib.import_module("PyPDF2")
|
||||
except ImportError:
|
||||
_out("Warning: PyPDF2 not installed, cannot read PDF files")
|
||||
return ""
|
||||
reader = pypdf2.PdfReader(filepath)
|
||||
return " ".join(page.extract_text() or "" for page in reader.pages)
|
||||
|
||||
|
||||
def _read_docx(filepath: str) -> str:
|
||||
try:
|
||||
docx = importlib.import_module("docx")
|
||||
except ImportError:
|
||||
_out("Warning: python-docx not installed, cannot read DOCX files")
|
||||
return ""
|
||||
doc = docx.Document(filepath)
|
||||
return " ".join(para.text for para in doc.paragraphs)
|
||||
|
||||
|
||||
def read_file(filepath: str) -> str:
|
||||
"""Read text from a .pdf, .docx, or plain-text file."""
|
||||
path = Path(filepath)
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
if suffix == ".pdf":
|
||||
return _read_pdf(filepath)
|
||||
if suffix == ".docx":
|
||||
return _read_docx(filepath)
|
||||
with path.open(encoding="utf-8", errors="ignore") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def preprocess_text(text: str) -> str:
|
||||
"""Lowercase, tokenize, and strip stopwords/non-alphanumerics."""
|
||||
text = text.lower()
|
||||
try:
|
||||
stop_words = set(stopwords.words("english"))
|
||||
words = word_tokenize(text)
|
||||
words = [w for w in words if w.isalnum() and w not in stop_words]
|
||||
return " ".join(words)
|
||||
except LookupError:
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def compute_similarity_matrix(documents: list[str]) -> list[list[float]]:
|
||||
"""Compute the TF-IDF cosine similarity matrix for a set of documents.
|
||||
|
||||
Returns a plain nested list, not the numpy array `cosine_similarity`
|
||||
produces: numpy is a plagiarism-toolchain dependency the installer
|
||||
pip-installs at runtime, deliberately absent from `meta/requirements.txt`,
|
||||
so mypy cannot follow it into this repo's type-checked surface
|
||||
(`no-any-unimported`). Converting at this boundary keeps every other
|
||||
signature in the module concretely typed.
|
||||
"""
|
||||
vectorizer = TfidfVectorizer(ngram_range=(1, 3), min_df=1, max_df=0.95)
|
||||
tfidf_matrix = vectorizer.fit_transform(documents)
|
||||
matrix: list[list[float]] = cosine_similarity(tfidf_matrix).tolist()
|
||||
return matrix
|
||||
|
||||
|
||||
def find_similar_passages(
|
||||
text1: str, text2: str, min_words: int = 5
|
||||
) -> list[tuple[str, str, float]]:
|
||||
"""Find similar sentence-level passages between two texts."""
|
||||
sentences1 = [s for s in sent_tokenize(text1) if len(s.split()) >= min_words]
|
||||
sentences2 = [s for s in sent_tokenize(text2) if len(s.split()) >= min_words]
|
||||
|
||||
if not sentences1 or not sentences2:
|
||||
return []
|
||||
|
||||
all_sentences = sentences1 + sentences2
|
||||
preprocessed = [preprocess_text(s) for s in all_sentences]
|
||||
|
||||
try:
|
||||
vectorizer = TfidfVectorizer(ngram_range=(1, 2))
|
||||
tfidf_matrix = vectorizer.fit_transform(preprocessed)
|
||||
except ValueError:
|
||||
return []
|
||||
|
||||
n1 = len(sentences1)
|
||||
similarities = []
|
||||
|
||||
for i, s1 in enumerate(sentences1):
|
||||
for j, s2 in enumerate(sentences2):
|
||||
sim = cosine_similarity(
|
||||
tfidf_matrix[i : i + 1], tfidf_matrix[n1 + j : n1 + j + 1]
|
||||
)[0][0]
|
||||
if sim > PASSAGE_SIMILARITY_THRESHOLD:
|
||||
similarities.append((s1, s2, sim))
|
||||
|
||||
return sorted(similarities, key=lambda x: x[2], reverse=True)
|
||||
108
linux_configuration/scripts/single_use/utils/sherlock.py
Executable file
108
linux_configuration/scripts/single_use/utils/sherlock.py
Executable file
@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sherlock - simple text plagiarism detector using n-gram fingerprinting.
|
||||
|
||||
Extracted from `install_plagiarism_tools.sh`'s fallback path, which used to
|
||||
write this out via a `cat <<'SHERLOCKEOF'` heredoc when cloning the real
|
||||
sherlock-py failed. Inline heredoc Python is invisible to ruff, mypy, pylint,
|
||||
bandit and pytest, so it lives in its own module instead, matching
|
||||
`download_nltk_data.py`'s extraction.
|
||||
|
||||
The installer copies this file into `$SHERLOCK_DIR/sherlock.py` only when
|
||||
`git clone` of the upstream tool fails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
from itertools import combinations
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
#: Minimum number of input files for a pairwise comparison to make sense.
|
||||
MIN_FILES = 2
|
||||
|
||||
|
||||
def _out(line: str = "") -> None:
|
||||
sys.stdout.write(line + "\n")
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""Split text into lowercase alphanumeric words."""
|
||||
return [w.lower() for w in text.split() if w.isalnum()]
|
||||
|
||||
|
||||
def get_ngrams(tokens: list[str], n: int = 3) -> list[tuple[str, ...]]:
|
||||
"""Generate n-grams from a token list."""
|
||||
return [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)]
|
||||
|
||||
|
||||
def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set[str]:
|
||||
"""Create a document fingerprint from sampled n-gram hashes."""
|
||||
tokens = tokenize(text)
|
||||
ngrams = get_ngrams(tokens, ngram_size)
|
||||
|
||||
fingerprints = set()
|
||||
for i, ng in enumerate(ngrams):
|
||||
if i % sample_rate == 0:
|
||||
h = hashlib.md5("".join(ng).encode(), usedforsecurity=False).hexdigest()[:8]
|
||||
fingerprints.add(h)
|
||||
|
||||
return fingerprints
|
||||
|
||||
|
||||
def compare_documents(fp1: set[str], fp2: set[str]) -> float:
|
||||
"""Jaccard similarity between two fingerprints."""
|
||||
if not fp1 or not fp2:
|
||||
return 0.0
|
||||
intersection = len(fp1 & fp2)
|
||||
union = len(fp1 | fp2)
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def read_document(filepath: str) -> str:
|
||||
"""Read a document's raw text content."""
|
||||
with Path(filepath).open(encoding="utf-8", errors="ignore") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Fingerprint every input file and report pairwise similarity."""
|
||||
parser = argparse.ArgumentParser(description="Sherlock - Text Plagiarism Detector")
|
||||
parser.add_argument("files", nargs="+", help="Files to compare")
|
||||
parser.add_argument(
|
||||
"--ngram", "-n", type=int, default=3, help="N-gram size (default: 3)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--threshold", "-t", type=float, default=0.1, help="Similarity threshold"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(args.files) < MIN_FILES:
|
||||
_out("Need at least 2 files to compare")
|
||||
return 1
|
||||
|
||||
docs = {}
|
||||
for f in args.files:
|
||||
if Path(f).exists():
|
||||
text = read_document(f)
|
||||
docs[f] = fingerprint(text, args.ngram)
|
||||
|
||||
_out("\nSherlock Plagiarism Analysis")
|
||||
_out("=" * 50)
|
||||
|
||||
files = list(docs.keys())
|
||||
for file1, file2 in combinations(files, 2):
|
||||
sim = compare_documents(docs[file1], docs[file2])
|
||||
name1 = Path(file1).name
|
||||
name2 = Path(file2).name
|
||||
flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else ""
|
||||
_out(f"{name1} <-> {name2}: {sim:.1%}{flag}")
|
||||
|
||||
_out("=" * 50)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Reference in New Issue
Block a user