mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 20:07:04 +01:00
Archive install_plagiarism_tools.sh: unused, moved to testsAndMisc-archive
Full history preserved via git filter-repo, including the pre-rename path from before the single_use/periodic_background split. Live at github.com/kuhyx/testsAndMisc-archive/tree/main/linux_configuration/scripts/single_use/utils. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
This commit is contained in:
parent
4e17ce2fd1
commit
071d706d8a
@ -0,0 +1,16 @@
|
||||
{
|
||||
"title": "Move install_plagiarism_tools.sh and siblings to testsAndMisc-archive",
|
||||
"objective": "The user said they no longer use the plagiarism-detection tooling (built and just split this session per NEXT_SESSION_PROMPT.md §3). Move it out of testsAndMisc into testsAndMisc-archive with full git history preserved, rather than leaving unused code in the primary repo.",
|
||||
"acceptance_criteria": [
|
||||
"All 13 plagiarism-tool files (install_plagiarism_tools.sh, download_nltk_data.py, check_plagiarism.py, plagiarism_similarity.py, sherlock.py, the 3 lib/plagiarism_*.sh siblings, and the 5 lib/tests/*plagiarism* test files) exist in kuhyx/testsAndMisc-archive with their git history intact, including the pre-rename path (linux_configuration/scripts/utils/install_plagiarism_tools.sh) from before the single_use/periodic_background reorg.",
|
||||
"The archive push completed and is verified live on GitHub before any deletion happens in the main repo -- deleting first and discovering a failed archive push would lose the only copy.",
|
||||
"The same 13 files are removed from testsAndMisc via git rm, with no other files touched.",
|
||||
"check_file_length.sh --all reports the same 14 over-cap files as immediately after the split commit (2a7b9d9) -- this deletion does not change that count, since none of the removed files were over cap.",
|
||||
"pre-commit run --files passes on the deletion commit's staged set."
|
||||
],
|
||||
"out_of_scope": [
|
||||
"Any other unused or academic-tooling files (generate_study_materials.sh, repo_to_study.sh, etc.) -- user explicitly scoped this to just the plagiarism-tool files.",
|
||||
"Re-running or re-verifying the split work itself; that was already proven correct before the archival decision was made (see install-plagiarism-tools-split-2026-08-18.json)."
|
||||
],
|
||||
"verifier": "gh api repos/kuhyx/testsAndMisc-archive/contents/linux_configuration/scripts/single_use/utils confirming all 13 files present before the git rm commit; git log --oneline -- <each removed path> in testsAndMisc-archive confirming history depth matches the pre-archival testsAndMisc log; check_file_length.sh --all; pre-commit run --files on the deletion commit."
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
{
|
||||
"intent": "Move the just-split, unused plagiarism-detection tooling (install_plagiarism_tools.sh and its 12 extracted/test siblings) out of testsAndMisc into testsAndMisc-archive with full git history, per the user's explicit statement that they no longer use it.",
|
||||
"scope": [
|
||||
"linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh, download_nltk_data.py, check_plagiarism.py, plagiarism_similarity.py, sherlock.py (deleted)",
|
||||
"linux_configuration/scripts/single_use/utils/lib/plagiarism_python.sh, plagiarism_sherlock.sh, plagiarism_optional.sh (deleted)",
|
||||
"linux_configuration/scripts/single_use/utils/lib/tests/plagiarism_harness.sh, test_plagiarism_python.sh, test_plagiarism_sherlock.sh, test_plagiarism_optional.sh, test_install_plagiarism_tools.sh (deleted)",
|
||||
"kuhyx/testsAndMisc-archive: same 13 files added, with history",
|
||||
"Not in scope: any other unused academic-tooling file the user did not name (e.g. generate_study_materials.sh)"
|
||||
],
|
||||
"changes": [
|
||||
"Cloned testsAndMisc into a throwaway scratchpad location (never a worktree of the real repo, since git filter-repo rewrites refs and a worktree shares the object store).",
|
||||
"Ran git filter-repo --force with explicit --path for all 13 current-path files plus the pre-rename path (linux_configuration/scripts/utils/install_plagiarism_tools.sh, from before the single_use/periodic_background reorg) so history predating that rename was not silently dropped.",
|
||||
"Cloned testsAndMisc-archive separately, added the filtered clone as a remote, and merged it in with --allow-unrelated-histories (required since the two repos share no common root commit).",
|
||||
"Verified the merge landed all 13 files with git ls-tree, then pushed to testsAndMisc-archive and confirmed live via the GitHub API before touching the main repo.",
|
||||
"Only after the archive push was confirmed live: git rm the same 13 files from testsAndMisc and committed the deletion.",
|
||||
"Removed the plag remote from the archive-clone before pushing, so no stray remote-tracking branches from the filtered clone's other refs (copilot-wt, etc.) leaked into the push."
|
||||
],
|
||||
"verification": [
|
||||
{
|
||||
"command": "gh api repos/kuhyx/testsAndMisc-archive/contents/linux_configuration/scripts/single_use/utils",
|
||||
"result": "pass",
|
||||
"evidence": "Returned check_plagiarism.py, download_nltk_data.py, install_plagiarism_tools.sh, lib, plagiarism_similarity.py, sherlock.py -- confirmed live on GitHub before the main-repo deletion commit was made."
|
||||
},
|
||||
{
|
||||
"command": "git log --oneline -- linux_configuration/scripts/single_use/utils/install_plagiarism_tools.sh (in the archive clone) vs the same command in testsAndMisc at commit 2a7b9d9",
|
||||
"result": "pass",
|
||||
"evidence": "Both show 4 commits with identical messages in the same order (the split, the no-inline-Python gate, the shellcheck fail-build change, the single_use/periodic_background rename) -- history genuinely carried over, not truncated at the rename."
|
||||
},
|
||||
{
|
||||
"command": "bash meta/scripts/check_file_length.sh --all",
|
||||
"result": "pass",
|
||||
"evidence": "14 files over cap, unchanged from immediately after the split commit -- none of the deleted files were over cap, so this operation does not move that number."
|
||||
}
|
||||
],
|
||||
"risks": [
|
||||
"The archive repo has no pre-commit hooks or CI, so nothing re-verifies the moved code still works there -- it is frozen at exactly the state it left testsAndMisc in.",
|
||||
"git filter-repo was run with --force on a disposable clone rather than a truly fresh `git clone` from the remote; verified this clone's origin was the local repo path (harmless) and that no push was made to it, but noting the deviation from the tool's documented 'fresh clone' expectation."
|
||||
],
|
||||
"rollback": [
|
||||
"In testsAndMisc: git revert the deletion commit to restore the 13 files (their content and permissions are unchanged from the split commit, so a straight revert is safe).",
|
||||
"The testsAndMisc-archive copy can stay regardless of rollback -- having the files in both places briefly is not harmful, just redundant. If it must be undone: git revert the merge commit there, though a merge revert needs the -m flag to pick which parent to keep."
|
||||
]
|
||||
}
|
||||
@ -1,165 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare documents for similarity using TF-IDF cosine similarity.
|
||||
|
||||
Extracted from `install_plagiarism_tools.sh`, which used to write this out via
|
||||
a `cat <<'PYEOF'` heredoc. Inline heredoc Python is invisible to ruff, mypy,
|
||||
pylint, bandit and pytest, so it lives in its own module instead, matching
|
||||
`download_nltk_data.py`'s extraction. The comparison engine (file reading,
|
||||
preprocessing, TF-IDF scoring) lives in `plagiarism_similarity.py`, split out
|
||||
to keep both files under the 250-line cap.
|
||||
|
||||
The installer copies this file into `$INSTALL_DIR/check_plagiarism.py` and
|
||||
wires the `plagcheck` wrapper to run it inside the venv.
|
||||
|
||||
Usage:
|
||||
python check_plagiarism.py file1.txt file2.txt [file3.txt ...]
|
||||
python check_plagiarism.py --dir /path/to/documents/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from plagiarism_similarity import (
|
||||
compute_similarity_matrix,
|
||||
find_similar_passages,
|
||||
preprocess_text,
|
||||
read_file,
|
||||
)
|
||||
|
||||
#: Minimum number of documents needed for a pairwise comparison to make sense.
|
||||
MIN_DOCUMENTS = 2
|
||||
|
||||
|
||||
def _out(line: str = "") -> None:
|
||||
sys.stdout.write(line + "\n")
|
||||
|
||||
|
||||
def _collect_files(args: argparse.Namespace) -> list[str]:
|
||||
files: list[str] = list(args.files)
|
||||
if args.dir:
|
||||
dir_path = Path(args.dir)
|
||||
for ext in ("*.txt", "*.pdf", "*.docx", "*.md", "*.tex"):
|
||||
files.extend(str(f) for f in dir_path.glob(ext))
|
||||
return files
|
||||
|
||||
|
||||
def _read_documents(files: list[str]) -> tuple[list[str], list[str]]:
|
||||
documents: list[str] = []
|
||||
filenames: list[str] = []
|
||||
for f in files:
|
||||
if not Path(f).exists():
|
||||
_out(f"Warning: {f} does not exist")
|
||||
continue
|
||||
text = read_file(f)
|
||||
if text.strip():
|
||||
documents.append(preprocess_text(text))
|
||||
filenames.append(Path(f).name)
|
||||
else:
|
||||
_out(f"Warning: {f} is empty or unreadable")
|
||||
return documents, filenames
|
||||
|
||||
|
||||
def _report_pairs(
|
||||
documents: list[str],
|
||||
filenames: list[str],
|
||||
sim_matrix: list[list[float]],
|
||||
threshold: float,
|
||||
) -> list[tuple[int, int, float, str]]:
|
||||
_out(f"{'Document Pair':<50} {'Similarity':>12}")
|
||||
_out("-" * 62)
|
||||
|
||||
suspicious_pairs: list[tuple[int, int, float, str]] = []
|
||||
for i in range(len(documents)):
|
||||
for j in range(i + 1, len(documents)):
|
||||
similarity = sim_matrix[i][j]
|
||||
pair_name = f"{filenames[i]} <-> {filenames[j]}"
|
||||
if similarity >= threshold:
|
||||
suspicious_pairs.append((i, j, similarity, pair_name))
|
||||
_out(f"{pair_name:<50} {similarity:>10.1%} ⚠️")
|
||||
else:
|
||||
_out(f"{pair_name:<50} {similarity:>10.1%}")
|
||||
|
||||
_out("-" * 62)
|
||||
return suspicious_pairs
|
||||
|
||||
|
||||
def _print_detailed_passages(
|
||||
files: list[str], suspicious_pairs: list[tuple[int, int, float, str]]
|
||||
) -> None:
|
||||
_out("\n" + "=" * 60)
|
||||
_out(" Detailed Similar Passages")
|
||||
_out("=" * 60)
|
||||
|
||||
for i, j, sim, pair_name in suspicious_pairs[:3]:
|
||||
_out(f"\n{pair_name} ({sim:.1%} similar):")
|
||||
_out("-" * 40)
|
||||
|
||||
raw_docs = [read_file(files[i]), read_file(files[j])]
|
||||
passages = find_similar_passages(raw_docs[0], raw_docs[1])
|
||||
|
||||
for s1, s2, psim in passages[:5]:
|
||||
_out(f'\n[{psim:.0%}] Document 1: "{s1[:100]}..."')
|
||||
_out(f' Document 2: "{s2[:100]}..."')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Compare every pair of input documents and report suspicious pairs."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Text Plagiarism Checker - Compare documents for similarity"
|
||||
)
|
||||
parser.add_argument("files", nargs="*", help="Files to compare")
|
||||
parser.add_argument("--dir", "-d", help="Directory containing documents to compare")
|
||||
parser.add_argument(
|
||||
"--threshold",
|
||||
"-t",
|
||||
type=float,
|
||||
default=0.3,
|
||||
help="Similarity threshold for flagging (0-1, default: 0.3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--detailed", "-v", action="store_true", help="Show detailed similar passages"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
files = _collect_files(args)
|
||||
if len(files) < MIN_DOCUMENTS:
|
||||
_out("Error: Need at least 2 files to compare")
|
||||
parser.print_help()
|
||||
return 1
|
||||
|
||||
_out(f"\n{'=' * 60}")
|
||||
_out(f" Plagiarism Check - Analyzing {len(files)} documents")
|
||||
_out(f"{'=' * 60}\n")
|
||||
|
||||
documents, filenames = _read_documents(files)
|
||||
if len(documents) < MIN_DOCUMENTS:
|
||||
_out("Error: Not enough valid documents to compare")
|
||||
return 1
|
||||
|
||||
_out("Computing document similarities...\n")
|
||||
sim_matrix = compute_similarity_matrix(documents)
|
||||
|
||||
suspicious_pairs = _report_pairs(documents, filenames, sim_matrix, args.threshold)
|
||||
|
||||
if suspicious_pairs:
|
||||
_out(
|
||||
f"\n⚠️ {len(suspicious_pairs)} pair(s) exceed "
|
||||
f"{args.threshold:.0%} similarity threshold\n"
|
||||
)
|
||||
if args.detailed:
|
||||
_print_detailed_passages(files, suspicious_pairs)
|
||||
else:
|
||||
_out(f"\n✓ No document pairs exceed {args.threshold:.0%} similarity threshold")
|
||||
|
||||
_out("\n" + "=" * 60)
|
||||
_out(" Analysis complete")
|
||||
_out("=" * 60 + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -1,53 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download the NLTK corpora the plagiarism tools need.
|
||||
|
||||
Extracted from `install_plagiarism_tools.sh`, which used to run this as an
|
||||
inline `python3 -c` block. Inline Python is invisible to ruff, mypy, pylint,
|
||||
bandit and pytest, so it lives in its own module instead.
|
||||
|
||||
`nltk` is a dependency of the plagiarism toolchain, not of this repo, so it is
|
||||
absent from `meta/requirements.txt` on purpose. The installer script pip-installs
|
||||
it immediately before invoking this module, and probes for it first.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
#: Corpora required by the plagiarism checker. `punkt_tab` is the post-3.8.2
|
||||
#: replacement for `punkt`; both are listed so the script works on either.
|
||||
CORPORA = (
|
||||
"punkt",
|
||||
"stopwords",
|
||||
"punkt_tab",
|
||||
"averaged_perceptron_tagger",
|
||||
"wordnet",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Download every corpus, reporting any that fail.
|
||||
|
||||
Returns 1 if any download failed, so the calling shell script's `set -e`
|
||||
sees a real failure instead of a silent partial install.
|
||||
"""
|
||||
# Imported dynamically, not at module scope: nltk belongs to the plagiarism
|
||||
# toolchain rather than this repo, so it is deliberately absent from
|
||||
# meta/requirements.txt and unresolvable to static analysis. The installer
|
||||
# pip-installs it immediately before invoking this module.
|
||||
try:
|
||||
nltk = importlib.import_module("nltk")
|
||||
except ImportError:
|
||||
sys.stderr.write("error: nltk is not installed\n")
|
||||
return 1
|
||||
|
||||
failed = [corpus for corpus in CORPORA if not nltk.download(corpus)]
|
||||
if failed:
|
||||
sys.stderr.write(f"error: failed to download: {', '.join(failed)}\n")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -1,94 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install Free & Open Source Plagiarism Detection Tools for Text
|
||||
# Suitable for academic work (theses, papers, etc.)
|
||||
#
|
||||
# Tools installed:
|
||||
# 1. Python NLP-based similarity detection (sklearn, NLTK, spaCy)
|
||||
# 2. Sherlock text plagiarism detector
|
||||
# 3. Ferret (Java-based, if Java available)
|
||||
# 4. Optional: WCopyfind via Wine (Windows tool)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
INSTALL_DIR="${HOME}/.local/share/plagiarism-tools"
|
||||
VENV_DIR="${HOME}/.local/share/plagiarism-venv"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
success() { echo -e "${GREEN}✓ $1${NC}"; }
|
||||
warn() { echo -e "${YELLOW}⚠ $1${NC}"; }
|
||||
error() { echo -e "${RED}✗ $1${NC}"; }
|
||||
|
||||
SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
|
||||
# shellcheck source=lib/plagiarism_python.sh
|
||||
source "$SCRIPT_DIR/lib/plagiarism_python.sh"
|
||||
# shellcheck source=lib/plagiarism_sherlock.sh
|
||||
source "$SCRIPT_DIR/lib/plagiarism_sherlock.sh"
|
||||
# shellcheck source=lib/plagiarism_optional.sh
|
||||
source "$SCRIPT_DIR/lib/plagiarism_optional.sh"
|
||||
|
||||
print_summary() {
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo " Installation Complete!"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
echo "Installed tools:"
|
||||
echo ""
|
||||
echo "1. Python NLP Plagiarism Checker (TF-IDF, cosine similarity)"
|
||||
echo " Usage: plagcheck file1.txt file2.txt"
|
||||
echo " plagcheck --dir /path/to/documents/ --detailed"
|
||||
echo " Location: $INSTALL_DIR/check_plagiarism.py"
|
||||
echo ""
|
||||
echo "2. Sherlock (n-gram fingerprinting)"
|
||||
echo " Location: $INSTALL_DIR/sherlock/sherlock.py"
|
||||
echo ""
|
||||
echo "3. Python virtual environment with NLP libraries:"
|
||||
echo " - scikit-learn (TF-IDF, cosine similarity)"
|
||||
echo " - NLTK (tokenization, stopwords)"
|
||||
echo " - spaCy (NLP processing)"
|
||||
echo " - gensim (document similarity)"
|
||||
echo " - textdistance, fuzzywuzzy (string matching)"
|
||||
echo " Activate with: source $VENV_DIR/bin/activate"
|
||||
echo ""
|
||||
echo "Quick Start:"
|
||||
echo " plagcheck thesis_v1.pdf thesis_v2.pdf --detailed"
|
||||
echo " plagcheck --dir ./student_papers/ --threshold 0.4"
|
||||
echo ""
|
||||
echo "Note: Ensure ~/.local/bin is in your PATH:"
|
||||
# Printed for the user to paste into their shell rc: $HOME and $PATH must
|
||||
# stay literal rather than resolve to this machine's values.
|
||||
# shellcheck disable=SC2016
|
||||
echo ' export PATH="$HOME/.local/bin:$PATH"'
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
|
||||
if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then
|
||||
warn "Add ~/.local/bin to your PATH by adding this to ~/.bashrc or ~/.zshrc:"
|
||||
# shellcheck disable=SC2016
|
||||
echo ' export PATH="$HOME/.local/bin:$PATH"'
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
echo "=============================================="
|
||||
echo " Open Source Plagiarism Detection Installer"
|
||||
echo " For Academic Text (Theses, Papers, etc.)"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
|
||||
install_python_nlp_tools "$SCRIPT_DIR"
|
||||
install_sherlock "$SCRIPT_DIR"
|
||||
check_ferret
|
||||
check_wcopyfind
|
||||
|
||||
print_summary
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@ -1,48 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Stages 3-4: optional plagiarism tools that need a manual download.
|
||||
#
|
||||
# Sourced by install_plagiarism_tools.sh; split out to keep it under the
|
||||
# 250-line cap. Sourced rather than run, so it inherits the caller's strict
|
||||
# mode and the color/log helpers and INSTALL_DIR global defined above the
|
||||
# source line.
|
||||
|
||||
# Ferret (Java-based) - Optional
|
||||
check_ferret() {
|
||||
echo ""
|
||||
echo "=== 3. Checking for Ferret (Java-based plagiarism tool) ==="
|
||||
|
||||
if ! command -v java &>/dev/null; then
|
||||
warn "Java not installed, skipping Ferret"
|
||||
return
|
||||
fi
|
||||
|
||||
local ferret_dir="$INSTALL_DIR/ferret"
|
||||
if [ -d "$ferret_dir" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
mkdir -p "$ferret_dir"
|
||||
echo "Ferret is a Java-based tool from University of Hertfordshire."
|
||||
echo "Download manually from: https://homepages.herts.ac.uk/~comqcln/Ferret/"
|
||||
echo "Place JAR file in: $ferret_dir"
|
||||
warn "Ferret requires manual download (academic license)"
|
||||
}
|
||||
|
||||
# WCopyfind via Wine (Optional)
|
||||
check_wcopyfind() {
|
||||
echo ""
|
||||
echo "=== 4. WCopyfind Information (Windows tool, needs Wine) ==="
|
||||
|
||||
if command -v wine &>/dev/null; then
|
||||
echo "Wine is available. WCopyfind can be run via Wine."
|
||||
echo "Download from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/"
|
||||
echo "Run with: wine /path/to/WCopyfind.exe"
|
||||
warn "WCopyfind requires manual download"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Wine not installed. To use WCopyfind:"
|
||||
echo " 1. Install wine: sudo apt install wine (or equivalent)"
|
||||
echo " 2. Download WCopyfind from: https://plagiarism.bloomfieldmedia.com/software/wcopyfind/"
|
||||
warn "WCopyfind skipped (Wine not available)"
|
||||
}
|
||||
@ -1,80 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Stage 1: Python NLP-based plagiarism detection environment.
|
||||
#
|
||||
# Sourced by install_plagiarism_tools.sh; split out to keep it under the
|
||||
# 250-line cap. Sourced rather than run, so it inherits the caller's strict
|
||||
# mode and the color/log helpers and INSTALL_DIR/VENV_DIR globals defined
|
||||
# above the source line.
|
||||
|
||||
# Create the venv, pip-install the NLP stack, and install check_plagiarism.py
|
||||
# (plus the plagiarism_similarity.py module it imports) and its "plagcheck"
|
||||
# wrapper. $1 is the directory install_plagiarism_tools.sh lives in, so both
|
||||
# scripts can be copied from beside it.
|
||||
install_python_nlp_tools() {
|
||||
local script_dir="$1"
|
||||
|
||||
echo ""
|
||||
echo "=== 1. Installing Python NLP-based Plagiarism Tools ==="
|
||||
|
||||
if ! command -v python3 &>/dev/null; then
|
||||
error "Python 3 is required but not installed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$VENV_DIR" ]; then
|
||||
echo "Creating Python virtual environment..."
|
||||
python3 -m venv "$VENV_DIR"
|
||||
success "Virtual environment created at $VENV_DIR"
|
||||
else
|
||||
warn "Virtual environment already exists at $VENV_DIR"
|
||||
fi
|
||||
|
||||
# Created at runtime by `python -m venv` above, so there is nothing on disk
|
||||
# for the linter to follow at check time.
|
||||
# shellcheck disable=SC1091
|
||||
source "$VENV_DIR/bin/activate"
|
||||
|
||||
echo "Installing Python packages for text similarity detection..."
|
||||
pip install --upgrade pip
|
||||
|
||||
pip install --progress-bar on \
|
||||
scikit-learn \
|
||||
nltk \
|
||||
spacy \
|
||||
gensim \
|
||||
numpy \
|
||||
pandas \
|
||||
python-docx \
|
||||
PyPDF2 \
|
||||
beautifulsoup4 \
|
||||
lxml \
|
||||
textdistance \
|
||||
fuzzywuzzy \
|
||||
python-Levenshtein
|
||||
|
||||
success "Python NLP packages installed"
|
||||
|
||||
echo "Downloading NLTK data (stopwords, punkt tokenizer)..."
|
||||
python3 "$script_dir/download_nltk_data.py"
|
||||
success "NLTK data downloaded"
|
||||
|
||||
echo "Downloading spaCy English model..."
|
||||
python3 -m spacy download en_core_web_sm 2>/dev/null || warn "spaCy model download may need manual install: python -m spacy download en_core_web_sm"
|
||||
success "spaCy model installed"
|
||||
|
||||
install -m 755 "$script_dir/check_plagiarism.py" "$INSTALL_DIR/check_plagiarism.py"
|
||||
install -m 644 "$script_dir/plagiarism_similarity.py" "$INSTALL_DIR/plagiarism_similarity.py"
|
||||
success "Created plagiarism checker script at $INSTALL_DIR/check_plagiarism.py"
|
||||
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
cat >"$HOME/.local/bin/plagcheck" <<WRAPEOF
|
||||
#!/usr/bin/env bash
|
||||
# Wrapper for plagiarism checker
|
||||
source "$VENV_DIR/bin/activate"
|
||||
python "$INSTALL_DIR/check_plagiarism.py" "\$@"
|
||||
WRAPEOF
|
||||
chmod +x "$HOME/.local/bin/plagcheck"
|
||||
success "Created 'plagcheck' command in ~/.local/bin/"
|
||||
|
||||
deactivate
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Stage 2: Sherlock text plagiarism detector.
|
||||
#
|
||||
# Sourced by install_plagiarism_tools.sh; split out to keep it under the
|
||||
# 250-line cap. Sourced rather than run, so it inherits the caller's strict
|
||||
# mode and the color/log helpers and INSTALL_DIR global defined above the
|
||||
# source line.
|
||||
|
||||
# Clone the upstream sherlock-py tool, falling back to the bundled
|
||||
# sherlock.py (n-gram fingerprinting) when git or the clone is unavailable.
|
||||
# $1 is the directory install_plagiarism_tools.sh lives in, so the fallback
|
||||
# sherlock.py can be copied from beside it.
|
||||
install_sherlock() {
|
||||
local script_dir="$1"
|
||||
|
||||
echo ""
|
||||
echo "=== 2. Installing Sherlock Text Plagiarism Detector ==="
|
||||
|
||||
local sherlock_dir="$INSTALL_DIR/sherlock"
|
||||
if [ -d "$sherlock_dir" ]; then
|
||||
warn "Sherlock already installed at $sherlock_dir"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! command -v git &>/dev/null; then
|
||||
warn "Git not available, skipping Sherlock installation"
|
||||
return
|
||||
fi
|
||||
|
||||
# There are several Sherlock implementations; using a popular Python one
|
||||
if git clone --depth 1 https://github.com/Zedeldi/sherlock-py.git "$sherlock_dir" 2>/dev/null; then
|
||||
success "Sherlock installed at $sherlock_dir"
|
||||
return
|
||||
fi
|
||||
|
||||
warn "Could not clone sherlock-py, trying alternative..."
|
||||
mkdir -p "$sherlock_dir"
|
||||
install -m 755 "$script_dir/sherlock.py" "$sherlock_dir/sherlock.py"
|
||||
success "Sherlock installed at $sherlock_dir"
|
||||
}
|
||||
@ -1,130 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/tests/plagiarism_harness.sh — fake python3/pip/git/java/wine behind the
|
||||
# install_plagiarism_tools.sh split.
|
||||
#
|
||||
# Sourced, not executed. Every external tool the installer calls is a shim on
|
||||
# PATH that records its invocation into $DEV and does the minimum real
|
||||
# filesystem work the calling code depends on (e.g. `python3 -m venv` must
|
||||
# leave a `bin/activate` a real `source` can run). INSTALL_DIR, VENV_DIR and
|
||||
# HOME all point into a throwaway tmpdir so nothing touches the real machine.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
UTILS_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
_t_pass() {
|
||||
PASS=$((PASS + 1))
|
||||
printf ' OK: %s\n' "$1"
|
||||
}
|
||||
|
||||
_t_fail() {
|
||||
FAIL=$((FAIL + 1))
|
||||
printf ' FAIL: %s\n' "$1"
|
||||
}
|
||||
|
||||
_t_eq() {
|
||||
local want="$1" got="$2" what="$3"
|
||||
if [[ "$got" == "$want" ]]; then
|
||||
_t_pass "$what"
|
||||
else
|
||||
_t_fail "$what (want '${want}', got '${got}')"
|
||||
fi
|
||||
}
|
||||
|
||||
TEST_TMPDIR="$(mktemp -d)"
|
||||
trap 'rm -rf "${TEST_TMPDIR}"' EXIT
|
||||
|
||||
readonly DEV="${TEST_TMPDIR}/device"
|
||||
readonly FAKE_BIN="${TEST_TMPDIR}/fake_bin"
|
||||
mkdir -p "${DEV}" "${FAKE_BIN}"
|
||||
|
||||
# --- fake external tools ----------------------------------------------------
|
||||
|
||||
cat >"${FAKE_BIN}/python3" <<'PYSHIM'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
DEV="${PLAGIARISM_TEST_DEV}"
|
||||
printf '%s\n' "python3 $*" >>"${DEV}/calls"
|
||||
if [[ "$1" == "-m" && "$2" == "venv" ]]; then
|
||||
venv_dir="$3"
|
||||
mkdir -p "${venv_dir}/bin"
|
||||
cat >"${venv_dir}/bin/activate" <<'ACTEOF'
|
||||
deactivate() { :; }
|
||||
pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; }
|
||||
ACTEOF
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$1" == "-m" && "$2" == "spacy" ]]; then
|
||||
[[ -f "${DEV}/fail_spacy" ]] && exit 1
|
||||
exit 0
|
||||
fi
|
||||
# python3 <script.py> [args...] — record which script ran and succeed.
|
||||
printf '%s\n' "ran:$1" >>"${DEV}/scripts_run"
|
||||
[[ -f "${DEV}/fail_$(basename "$1" .py)" ]] && exit 1
|
||||
exit 0
|
||||
PYSHIM
|
||||
chmod +x "${FAKE_BIN}/python3"
|
||||
|
||||
cat >"${FAKE_BIN}/git" <<'GITSHIM'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
DEV="${PLAGIARISM_TEST_DEV}"
|
||||
printf '%s\n' "git $*" >>"${DEV}/calls"
|
||||
if [[ "$1" == "clone" ]]; then
|
||||
[[ -f "${DEV}/fail_clone" ]] && exit 1
|
||||
dest="${*: -1}"
|
||||
mkdir -p "${dest}"
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
GITSHIM
|
||||
chmod +x "${FAKE_BIN}/git"
|
||||
|
||||
cat >"${FAKE_BIN}/java" <<'JAVASHIM'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
JAVASHIM
|
||||
chmod +x "${FAKE_BIN}/java"
|
||||
|
||||
cat >"${FAKE_BIN}/wine" <<'WINESHIM'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
WINESHIM
|
||||
chmod +x "${FAKE_BIN}/wine"
|
||||
|
||||
export PLAGIARISM_TEST_DEV="${DEV}"
|
||||
export PATH="${FAKE_BIN}:${PATH}"
|
||||
|
||||
# --- subject under test -----------------------------------------------------
|
||||
|
||||
HOME="${TEST_TMPDIR}/home"
|
||||
INSTALL_DIR="${TEST_TMPDIR}/install"
|
||||
VENV_DIR="${TEST_TMPDIR}/venv"
|
||||
mkdir -p "${HOME}" "${INSTALL_DIR}"
|
||||
|
||||
success() { :; }
|
||||
warn() { printf '%s\n' "$*" >>"${DEV}/warnings"; }
|
||||
error() { printf '%s\n' "$*" >>"${DEV}/errors"; }
|
||||
|
||||
# shellcheck source=../plagiarism_python.sh
|
||||
. "${UTILS_DIR}/lib/plagiarism_python.sh"
|
||||
# shellcheck source=../plagiarism_sherlock.sh
|
||||
. "${UTILS_DIR}/lib/plagiarism_sherlock.sh"
|
||||
# shellcheck source=../plagiarism_optional.sh
|
||||
. "${UTILS_DIR}/lib/plagiarism_optional.sh"
|
||||
|
||||
# reset_state — wipe INSTALL_DIR/VENV_DIR/DEV records between test groups so
|
||||
# each group starts from "nothing installed yet".
|
||||
reset_state() {
|
||||
rm -rf "${INSTALL_DIR}" "${VENV_DIR}" "${DEV:?}/calls" "${DEV}/scripts_run" \
|
||||
"${DEV}/warnings" "${DEV}/errors" 2>/dev/null || true
|
||||
mkdir -p "${INSTALL_DIR}"
|
||||
: >"${DEV}/calls"
|
||||
: >"${DEV}/scripts_run"
|
||||
: >"${DEV}/warnings"
|
||||
: >"${DEV}/errors"
|
||||
}
|
||||
reset_state
|
||||
@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end test for install_plagiarism_tools.sh's main()/print_summary(),
|
||||
# which stay in the entry script rather than a lib. Runs the real script as a
|
||||
# subprocess against the same fake python3/pip/git/java/wine as the lib tests,
|
||||
# with HOME pointed at a throwaway tmpdir so INSTALL_DIR/VENV_DIR (derived
|
||||
# from $HOME inside the script) land there too.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
readonly ENTRY_SCRIPT="${UTILS_DIR}/install_plagiarism_tools.sh"
|
||||
|
||||
# The entry script derives INSTALL_DIR/VENV_DIR from $HOME itself (a plain
|
||||
# assignment near its top), independently of the lib tests' INSTALL_DIR/
|
||||
# VENV_DIR harness variables. Running it as a real subprocess against the
|
||||
# harness's $HOME means its paths land under $HOME, not under the harness's
|
||||
# separate INSTALL_DIR/VENV_DIR -- so assertions below re-derive the same
|
||||
# paths the script itself computes, rather than reusing the harness ones.
|
||||
readonly ENTRY_INSTALL_DIR="${HOME}/.local/share/plagiarism-tools"
|
||||
readonly ENTRY_VENV_DIR="${HOME}/.local/share/plagiarism-venv"
|
||||
|
||||
# --- full run, PATH already in ~/.local/bin -------------------------------
|
||||
|
||||
mkdir -p "${HOME}/.local/bin"
|
||||
OUT_FULL="${TEST_TMPDIR}/out_full.log"
|
||||
PATH="${FAKE_BIN}:${HOME}/.local/bin:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_FULL}" 2>&1
|
||||
_status=$?
|
||||
|
||||
if [[ "${_status}" -eq 0 ]]; then
|
||||
_t_pass "install_plagiarism_tools.sh exits zero on a full successful run"
|
||||
else
|
||||
_t_fail "install_plagiarism_tools.sh should exit zero (log: $(cat "${OUT_FULL}"))"
|
||||
fi
|
||||
|
||||
if grep -q "Open Source Plagiarism Detection Installer" "${OUT_FULL}"; then
|
||||
_t_pass "main prints the installer banner"
|
||||
else
|
||||
_t_fail "main should print the installer banner"
|
||||
fi
|
||||
|
||||
if grep -q "=== 1. Installing Python NLP-based Plagiarism Tools ===" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 1 (Python NLP tools)"
|
||||
else
|
||||
_t_fail "main should run stage 1"
|
||||
fi
|
||||
|
||||
if grep -q "=== 2. Installing Sherlock Text Plagiarism Detector ===" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 2 (Sherlock)"
|
||||
else
|
||||
_t_fail "main should run stage 2"
|
||||
fi
|
||||
|
||||
if grep -q "=== 3. Checking for Ferret" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 3 (Ferret)"
|
||||
else
|
||||
_t_fail "main should run stage 3"
|
||||
fi
|
||||
|
||||
if grep -q "=== 4. WCopyfind Information" "${OUT_FULL}"; then
|
||||
_t_pass "main runs stage 4 (WCopyfind)"
|
||||
else
|
||||
_t_fail "main should run stage 4"
|
||||
fi
|
||||
|
||||
if grep -q "Installation Complete!" "${OUT_FULL}"; then
|
||||
_t_pass "main prints the summary"
|
||||
else
|
||||
_t_fail "main should print the summary"
|
||||
fi
|
||||
|
||||
if grep -q "Location: ${ENTRY_INSTALL_DIR}/check_plagiarism.py" "${OUT_FULL}"; then
|
||||
_t_pass "the summary reports the checker's installed location"
|
||||
else
|
||||
_t_fail "the summary should report the checker's installed location"
|
||||
fi
|
||||
|
||||
if grep -q "Location: ${ENTRY_INSTALL_DIR}/sherlock/sherlock.py" "${OUT_FULL}"; then
|
||||
_t_pass "the summary reports sherlock's installed location"
|
||||
else
|
||||
_t_fail "the summary should report sherlock's installed location"
|
||||
fi
|
||||
|
||||
if grep -q "Activate with: source ${ENTRY_VENV_DIR}/bin/activate" "${OUT_FULL}"; then
|
||||
_t_pass "the summary reports the venv activation command"
|
||||
else
|
||||
_t_fail "the summary should report the venv activation command"
|
||||
fi
|
||||
|
||||
# ~/.local/bin was already on PATH for this run, so the "add it to your PATH"
|
||||
# nudge must not repeat in the summary.
|
||||
if grep -q "Add ~/.local/bin to your PATH by adding this" "${OUT_FULL}"; then
|
||||
_t_fail "the summary should not repeat the PATH nudge when ~/.local/bin is already on PATH"
|
||||
else
|
||||
_t_pass "the summary skips the PATH nudge when ~/.local/bin is already on PATH"
|
||||
fi
|
||||
|
||||
# --- ~/.local/bin missing from PATH: the summary nudges once --------------
|
||||
|
||||
reset_state
|
||||
rm -rf "${HOME}/.local"
|
||||
OUT_NOPATH="${TEST_TMPDIR}/out_nopath.log"
|
||||
PATH="${FAKE_BIN}:${PATH}" bash "${ENTRY_SCRIPT}" >"${OUT_NOPATH}" 2>&1
|
||||
|
||||
if grep -qc "Add ~/.local/bin to your PATH by adding this" "${OUT_NOPATH}"; then
|
||||
_t_pass "the summary nudges to add ~/.local/bin to PATH when it is missing"
|
||||
else
|
||||
_t_fail "the summary should nudge to add ~/.local/bin to PATH when it is missing"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
@ -1,103 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for lib/plagiarism_optional.sh (Ferret/WCopyfind availability
|
||||
# checks) against the fake java/wine in plagiarism_harness.sh.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
_no_tool_path() {
|
||||
local skip="$1"
|
||||
local dir="${TEST_TMPDIR}/no_${skip}_bin"
|
||||
mkdir -p "${dir}"
|
||||
local tool
|
||||
for tool in python3 git java wine cat chmod mkdir; do
|
||||
[[ "${tool}" == "${skip}" ]] && continue
|
||||
[[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${dir}/${tool}"
|
||||
done
|
||||
printf '%s' "${dir}"
|
||||
}
|
||||
|
||||
# --- check_ferret: java available -----------------------------------------
|
||||
|
||||
check_ferret >/tmp/plagiarism_optional_test.log 2>&1
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/ferret" ]]; then
|
||||
_t_pass "check_ferret creates the ferret directory when java is available"
|
||||
else
|
||||
_t_fail "check_ferret should create ${INSTALL_DIR}/ferret when java is available"
|
||||
fi
|
||||
|
||||
if grep -q "requires manual download" "${DEV}/warnings"; then
|
||||
_t_pass "check_ferret explains that Ferret needs a manual download"
|
||||
else
|
||||
_t_fail "check_ferret should explain that Ferret needs a manual download"
|
||||
fi
|
||||
|
||||
# check_ferret creates the directory as a marker; a second run must not
|
||||
# re-print the manual-download instructions once it already exists.
|
||||
: >"${DEV}/warnings"
|
||||
check_ferret >/tmp/plagiarism_optional_test.log 2>&1
|
||||
if grep -q "requires manual download" "${DEV}/warnings"; then
|
||||
_t_fail "check_ferret should not repeat instructions once the ferret dir already exists"
|
||||
else
|
||||
_t_pass "check_ferret is a no-op once the ferret directory already exists"
|
||||
fi
|
||||
|
||||
# --- check_ferret: java unavailable ---------------------------------------
|
||||
|
||||
reset_state
|
||||
no_java_path="$(_no_tool_path java)"
|
||||
PATH="${no_java_path}" check_ferret >/dev/null 2>&1
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/ferret" ]]; then
|
||||
_t_fail "check_ferret should not create the ferret directory without java"
|
||||
else
|
||||
_t_pass "check_ferret skips Ferret entirely when java is unavailable"
|
||||
fi
|
||||
|
||||
if grep -q "Java not installed" "${DEV}/warnings"; then
|
||||
_t_pass "check_ferret warns when java is unavailable"
|
||||
else
|
||||
_t_fail "check_ferret should warn when java is unavailable"
|
||||
fi
|
||||
|
||||
# --- check_wcopyfind: wine available ---------------------------------------
|
||||
|
||||
: >"${DEV}/warnings"
|
||||
check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1
|
||||
|
||||
if grep -q "Wine is available" /tmp/plagiarism_optional_test.log; then
|
||||
_t_pass "check_wcopyfind reports Wine as available"
|
||||
else
|
||||
_t_fail "check_wcopyfind should report Wine as available"
|
||||
fi
|
||||
|
||||
if grep -q "requires manual download" "${DEV}/warnings"; then
|
||||
_t_pass "check_wcopyfind warns that WCopyfind needs a manual download"
|
||||
else
|
||||
_t_fail "check_wcopyfind should warn that WCopyfind needs a manual download"
|
||||
fi
|
||||
|
||||
# --- check_wcopyfind: wine unavailable --------------------------------------
|
||||
|
||||
: >"${DEV}/warnings"
|
||||
no_wine_path="$(_no_tool_path wine)"
|
||||
PATH="${no_wine_path}" check_wcopyfind >/tmp/plagiarism_optional_test.log 2>&1
|
||||
|
||||
if grep -q "Wine not installed" /tmp/plagiarism_optional_test.log; then
|
||||
_t_pass "check_wcopyfind reports Wine as unavailable"
|
||||
else
|
||||
_t_fail "check_wcopyfind should report Wine as unavailable"
|
||||
fi
|
||||
|
||||
if grep -q "skipped (Wine not available)" "${DEV}/warnings"; then
|
||||
_t_pass "check_wcopyfind warns that WCopyfind was skipped"
|
||||
else
|
||||
_t_fail "check_wcopyfind should warn that WCopyfind was skipped"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
@ -1,132 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for lib/plagiarism_python.sh against the fake python3/pip in
|
||||
# plagiarism_harness.sh. No real network or venv is needed.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
# --- happy path ---------------------------------------------------------
|
||||
|
||||
install_python_nlp_tools "${UTILS_DIR}" >/tmp/plagiarism_python_test.log 2>&1
|
||||
_status=$?
|
||||
if [[ "${_status}" -eq 0 ]]; then
|
||||
_t_pass "install_python_nlp_tools succeeds against a working toolchain"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should succeed (log: $(cat /tmp/plagiarism_python_test.log))"
|
||||
fi
|
||||
|
||||
if [[ -d "${VENV_DIR}" ]]; then
|
||||
_t_pass "install_python_nlp_tools creates the virtual environment"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should create ${VENV_DIR}"
|
||||
fi
|
||||
|
||||
if grep -q '^python3 -m venv' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools invokes python3 -m venv"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should invoke python3 -m venv"
|
||||
fi
|
||||
|
||||
if grep -q 'pip install --upgrade pip' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools upgrades pip"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should upgrade pip"
|
||||
fi
|
||||
|
||||
if grep -q 'scikit-learn' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools installs the NLP package set"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should install scikit-learn and friends"
|
||||
fi
|
||||
|
||||
if grep -q 'ran:.*download_nltk_data.py' "${DEV}/scripts_run"; then
|
||||
_t_pass "install_python_nlp_tools runs download_nltk_data.py"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should run download_nltk_data.py"
|
||||
fi
|
||||
|
||||
if grep -q '^python3 -m spacy download' "${DEV}/calls"; then
|
||||
_t_pass "install_python_nlp_tools downloads the spaCy model"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should download the spaCy model"
|
||||
fi
|
||||
|
||||
if [[ -x "${INSTALL_DIR}/check_plagiarism.py" ]]; then
|
||||
_t_pass "install_python_nlp_tools installs check_plagiarism.py, executable"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should install an executable check_plagiarism.py"
|
||||
fi
|
||||
|
||||
if [[ -f "${INSTALL_DIR}/plagiarism_similarity.py" ]]; then
|
||||
_t_pass "install_python_nlp_tools installs plagiarism_similarity.py alongside it"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should install plagiarism_similarity.py -- \
|
||||
check_plagiarism.py imports it, so a missing copy breaks at runtime, not install time"
|
||||
fi
|
||||
|
||||
if [[ -x "${HOME}/.local/bin/plagcheck" ]]; then
|
||||
_t_pass "install_python_nlp_tools creates an executable plagcheck wrapper"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should create an executable plagcheck wrapper"
|
||||
fi
|
||||
|
||||
if grep -q "${VENV_DIR}/bin/activate" "${HOME}/.local/bin/plagcheck"; then
|
||||
_t_pass "plagcheck wrapper activates the venv it was installed against"
|
||||
else
|
||||
_t_fail "plagcheck wrapper should reference ${VENV_DIR}/bin/activate"
|
||||
fi
|
||||
|
||||
if grep -q "${INSTALL_DIR}/check_plagiarism.py" "${HOME}/.local/bin/plagcheck"; then
|
||||
_t_pass "plagcheck wrapper points at the installed checker script"
|
||||
else
|
||||
_t_fail "plagcheck wrapper should point at ${INSTALL_DIR}/check_plagiarism.py"
|
||||
fi
|
||||
|
||||
# --- venv already exists: warn, don't recreate --------------------------
|
||||
|
||||
reset_state
|
||||
mkdir -p "${VENV_DIR}/bin"
|
||||
cat >"${VENV_DIR}/bin/activate" <<'ACTEOF'
|
||||
deactivate() { :; }
|
||||
pip() { printf '%s\n' "pip $*" >>"${PLAGIARISM_TEST_DEV}/calls"; return 0; }
|
||||
ACTEOF
|
||||
: >"${DEV}/calls"
|
||||
install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1
|
||||
if grep -q '^python3 -m venv' "${DEV}/calls"; then
|
||||
_t_fail "install_python_nlp_tools should not recreate an existing venv"
|
||||
else
|
||||
_t_pass "install_python_nlp_tools skips venv creation when one already exists"
|
||||
fi
|
||||
|
||||
# --- python3 missing: fatal, not silently skipped ------------------------
|
||||
|
||||
reset_state
|
||||
# A bare `rm -f "${FAKE_BIN}/python3"` does not hide python3 from
|
||||
# `command -v`: pyenv's shim is still reachable further down the real PATH.
|
||||
# Instead point PATH at a directory that shadows python3 with nothing before
|
||||
# falling through to the rest of the fakes, so `command -v python3` fails.
|
||||
NO_PYTHON_BIN="${TEST_TMPDIR}/no_python_bin"
|
||||
mkdir -p "${NO_PYTHON_BIN}"
|
||||
for tool in git java wine cat chmod mkdir; do
|
||||
[[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_PYTHON_BIN}/${tool}"
|
||||
done
|
||||
# install_python_nlp_tools calls `exit` (not `return`) on a missing python3,
|
||||
# by design -- a partial install with no interpreter must abort the whole
|
||||
# installer, not just this stage. Run it in a subshell so that exit only
|
||||
# ends the subshell, not this test script.
|
||||
if (PATH="${NO_PYTHON_BIN}" install_python_nlp_tools "${UTILS_DIR}" >/dev/null 2>&1); then
|
||||
_t_fail "install_python_nlp_tools should exit non-zero without python3"
|
||||
else
|
||||
_t_pass "install_python_nlp_tools exits non-zero when python3 is missing"
|
||||
fi
|
||||
if grep -q "Python 3 is required" "${DEV}/errors"; then
|
||||
_t_pass "install_python_nlp_tools reports the missing-python3 error"
|
||||
else
|
||||
_t_fail "install_python_nlp_tools should report the missing-python3 error"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
@ -1,106 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for lib/plagiarism_sherlock.sh against the fake git in
|
||||
# plagiarism_harness.sh. No real network is needed.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# shellcheck source=plagiarism_harness.sh
|
||||
. "${SCRIPT_DIR}/plagiarism_harness.sh"
|
||||
|
||||
# --- upstream clone succeeds ---------------------------------------------
|
||||
|
||||
install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if grep -q '^git clone' "${DEV}/calls"; then
|
||||
_t_pass "install_sherlock clones the upstream sherlock-py tool"
|
||||
else
|
||||
_t_fail "install_sherlock should attempt a git clone"
|
||||
fi
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/sherlock" ]]; then
|
||||
_t_pass "install_sherlock creates the sherlock directory on a successful clone"
|
||||
else
|
||||
_t_fail "install_sherlock should create ${INSTALL_DIR}/sherlock"
|
||||
fi
|
||||
|
||||
if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then
|
||||
_t_fail "a successful clone should not also install the bundled fallback sherlock.py"
|
||||
else
|
||||
_t_pass "a successful clone does not install the bundled fallback sherlock.py"
|
||||
fi
|
||||
|
||||
# --- already installed: no-op, warns -------------------------------------
|
||||
|
||||
: >"${DEV}/calls"
|
||||
install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if grep -q '^git clone' "${DEV}/calls"; then
|
||||
_t_fail "install_sherlock should not re-clone when already installed"
|
||||
else
|
||||
_t_pass "install_sherlock skips reinstalling when the directory already exists"
|
||||
fi
|
||||
|
||||
if grep -q "already installed" "${DEV}/warnings"; then
|
||||
_t_pass "install_sherlock warns that sherlock is already installed"
|
||||
else
|
||||
_t_fail "install_sherlock should warn that sherlock is already installed"
|
||||
fi
|
||||
|
||||
# --- clone fails: falls back to the bundled sherlock.py -------------------
|
||||
|
||||
reset_state
|
||||
touch "${DEV}/fail_clone"
|
||||
install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if [[ -f "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then
|
||||
_t_pass "install_sherlock falls back to the bundled sherlock.py when the clone fails"
|
||||
else
|
||||
_t_fail "install_sherlock should fall back to the bundled sherlock.py when the clone fails"
|
||||
fi
|
||||
|
||||
if [[ -x "${INSTALL_DIR}/sherlock/sherlock.py" ]]; then
|
||||
_t_pass "the bundled fallback sherlock.py is installed executable"
|
||||
else
|
||||
_t_fail "the bundled fallback sherlock.py should be installed executable"
|
||||
fi
|
||||
|
||||
if grep -q "Could not clone sherlock-py" "${DEV}/warnings"; then
|
||||
_t_pass "install_sherlock warns when the clone fails, before falling back"
|
||||
else
|
||||
_t_fail "install_sherlock should warn when the clone fails"
|
||||
fi
|
||||
|
||||
# The installed fallback must be byte-identical to the source file: it is the
|
||||
# one artifact the git-unavailable path can serve, so a stale copy would ship
|
||||
# broken plagiarism detection silently.
|
||||
if diff -q "${UTILS_DIR}/sherlock.py" "${INSTALL_DIR}/sherlock/sherlock.py" >/dev/null; then
|
||||
_t_pass "the installed fallback sherlock.py matches the source file byte-for-byte"
|
||||
else
|
||||
_t_fail "the installed fallback sherlock.py should match ${UTILS_DIR}/sherlock.py"
|
||||
fi
|
||||
|
||||
# --- git not available: warns, does not attempt to install ----------------
|
||||
|
||||
reset_state
|
||||
NO_GIT_BIN="${TEST_TMPDIR}/no_git_bin"
|
||||
mkdir -p "${NO_GIT_BIN}"
|
||||
for tool in python3 java wine cat chmod mkdir; do
|
||||
[[ -x "$(command -v "${tool}")" ]] && ln -sf "$(command -v "${tool}")" "${NO_GIT_BIN}/${tool}"
|
||||
done
|
||||
PATH="${NO_GIT_BIN}" install_sherlock "${UTILS_DIR}" >/dev/null 2>&1
|
||||
|
||||
if [[ -d "${INSTALL_DIR}/sherlock" ]]; then
|
||||
_t_fail "install_sherlock should not create the sherlock directory without git"
|
||||
else
|
||||
_t_pass "install_sherlock skips installation entirely when git is unavailable"
|
||||
fi
|
||||
|
||||
if grep -q "Git not available" "${DEV}/warnings"; then
|
||||
_t_pass "install_sherlock warns when git is unavailable"
|
||||
else
|
||||
_t_fail "install_sherlock should warn when git is unavailable"
|
||||
fi
|
||||
|
||||
printf '\nResults: %d passed, %d failed\n' "${PASS}" "${FAIL}"
|
||||
[[ "${FAIL}" -eq 0 ]]
|
||||
@ -1,141 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Document reading and TF-IDF similarity scoring for check_plagiarism.py.
|
||||
|
||||
Split out of `check_plagiarism.py` to keep it under the 250-line cap: this
|
||||
module holds the comparison engine (file reading, text preprocessing, and the
|
||||
TF-IDF/cosine-similarity computations), `check_plagiarism.py` keeps the CLI
|
||||
and reporting.
|
||||
|
||||
Its third-party imports (numpy, sklearn, nltk) are the plagiarism toolchain's
|
||||
dependencies, not this repo's, so they are absent from `meta/requirements.txt`
|
||||
on purpose. Importing them dynamically (rather than with a static `import
|
||||
nltk`) keeps this module importable -- and lintable -- in an environment
|
||||
that doesn't have them installed, matching `download_nltk_data.py`'s
|
||||
extraction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
#: Cosine similarity above this, for a single sentence pair, is worth reporting
|
||||
#: as a suspicious passage in --detailed mode.
|
||||
PASSAGE_SIMILARITY_THRESHOLD = 0.5
|
||||
|
||||
nltk = importlib.import_module("nltk")
|
||||
stopwords = importlib.import_module("nltk.corpus").stopwords
|
||||
nltk_tokenize = importlib.import_module("nltk.tokenize")
|
||||
sent_tokenize = nltk_tokenize.sent_tokenize
|
||||
word_tokenize = nltk_tokenize.word_tokenize
|
||||
TfidfVectorizer = importlib.import_module(
|
||||
"sklearn.feature_extraction.text"
|
||||
).TfidfVectorizer
|
||||
cosine_similarity = importlib.import_module(
|
||||
"sklearn.metrics.pairwise"
|
||||
).cosine_similarity
|
||||
|
||||
try:
|
||||
stopwords.words("english")
|
||||
except LookupError:
|
||||
nltk.download("stopwords", quiet=True)
|
||||
nltk.download("punkt", quiet=True)
|
||||
|
||||
|
||||
def _out(line: str = "") -> None:
|
||||
sys.stdout.write(line + "\n")
|
||||
|
||||
|
||||
def _read_pdf(filepath: str) -> str:
|
||||
try:
|
||||
pypdf2 = importlib.import_module("PyPDF2")
|
||||
except ImportError:
|
||||
_out("Warning: PyPDF2 not installed, cannot read PDF files")
|
||||
return ""
|
||||
reader = pypdf2.PdfReader(filepath)
|
||||
return " ".join(page.extract_text() or "" for page in reader.pages)
|
||||
|
||||
|
||||
def _read_docx(filepath: str) -> str:
|
||||
try:
|
||||
docx = importlib.import_module("docx")
|
||||
except ImportError:
|
||||
_out("Warning: python-docx not installed, cannot read DOCX files")
|
||||
return ""
|
||||
doc = docx.Document(filepath)
|
||||
return " ".join(para.text for para in doc.paragraphs)
|
||||
|
||||
|
||||
def read_file(filepath: str) -> str:
|
||||
"""Read text from a .pdf, .docx, or plain-text file."""
|
||||
path = Path(filepath)
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
if suffix == ".pdf":
|
||||
return _read_pdf(filepath)
|
||||
if suffix == ".docx":
|
||||
return _read_docx(filepath)
|
||||
with path.open(encoding="utf-8", errors="ignore") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def preprocess_text(text: str) -> str:
|
||||
"""Lowercase, tokenize, and strip stopwords/non-alphanumerics."""
|
||||
text = text.lower()
|
||||
try:
|
||||
stop_words = set(stopwords.words("english"))
|
||||
words = word_tokenize(text)
|
||||
words = [w for w in words if w.isalnum() and w not in stop_words]
|
||||
return " ".join(words)
|
||||
except LookupError:
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def compute_similarity_matrix(documents: list[str]) -> list[list[float]]:
|
||||
"""Compute the TF-IDF cosine similarity matrix for a set of documents.
|
||||
|
||||
Returns a plain nested list, not the numpy array `cosine_similarity`
|
||||
produces: numpy is a plagiarism-toolchain dependency the installer
|
||||
pip-installs at runtime, deliberately absent from `meta/requirements.txt`,
|
||||
so mypy cannot follow it into this repo's type-checked surface
|
||||
(`no-any-unimported`). Converting at this boundary keeps every other
|
||||
signature in the module concretely typed.
|
||||
"""
|
||||
vectorizer = TfidfVectorizer(ngram_range=(1, 3), min_df=1, max_df=0.95)
|
||||
tfidf_matrix = vectorizer.fit_transform(documents)
|
||||
matrix: list[list[float]] = cosine_similarity(tfidf_matrix).tolist()
|
||||
return matrix
|
||||
|
||||
|
||||
def find_similar_passages(
|
||||
text1: str, text2: str, min_words: int = 5
|
||||
) -> list[tuple[str, str, float]]:
|
||||
"""Find similar sentence-level passages between two texts."""
|
||||
sentences1 = [s for s in sent_tokenize(text1) if len(s.split()) >= min_words]
|
||||
sentences2 = [s for s in sent_tokenize(text2) if len(s.split()) >= min_words]
|
||||
|
||||
if not sentences1 or not sentences2:
|
||||
return []
|
||||
|
||||
all_sentences = sentences1 + sentences2
|
||||
preprocessed = [preprocess_text(s) for s in all_sentences]
|
||||
|
||||
try:
|
||||
vectorizer = TfidfVectorizer(ngram_range=(1, 2))
|
||||
tfidf_matrix = vectorizer.fit_transform(preprocessed)
|
||||
except ValueError:
|
||||
return []
|
||||
|
||||
n1 = len(sentences1)
|
||||
similarities = []
|
||||
|
||||
for i, s1 in enumerate(sentences1):
|
||||
for j, s2 in enumerate(sentences2):
|
||||
sim = cosine_similarity(
|
||||
tfidf_matrix[i : i + 1], tfidf_matrix[n1 + j : n1 + j + 1]
|
||||
)[0][0]
|
||||
if sim > PASSAGE_SIMILARITY_THRESHOLD:
|
||||
similarities.append((s1, s2, sim))
|
||||
|
||||
return sorted(similarities, key=lambda x: x[2], reverse=True)
|
||||
@ -1,108 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sherlock - simple text plagiarism detector using n-gram fingerprinting.
|
||||
|
||||
Extracted from `install_plagiarism_tools.sh`'s fallback path, which used to
|
||||
write this out via a `cat <<'SHERLOCKEOF'` heredoc when cloning the real
|
||||
sherlock-py failed. Inline heredoc Python is invisible to ruff, mypy, pylint,
|
||||
bandit and pytest, so it lives in its own module instead, matching
|
||||
`download_nltk_data.py`'s extraction.
|
||||
|
||||
The installer copies this file into `$SHERLOCK_DIR/sherlock.py` only when
|
||||
`git clone` of the upstream tool fails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
from itertools import combinations
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
#: Minimum number of input files for a pairwise comparison to make sense.
|
||||
MIN_FILES = 2
|
||||
|
||||
|
||||
def _out(line: str = "") -> None:
|
||||
sys.stdout.write(line + "\n")
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""Split text into lowercase alphanumeric words."""
|
||||
return [w.lower() for w in text.split() if w.isalnum()]
|
||||
|
||||
|
||||
def get_ngrams(tokens: list[str], n: int = 3) -> list[tuple[str, ...]]:
|
||||
"""Generate n-grams from a token list."""
|
||||
return [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)]
|
||||
|
||||
|
||||
def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set[str]:
|
||||
"""Create a document fingerprint from sampled n-gram hashes."""
|
||||
tokens = tokenize(text)
|
||||
ngrams = get_ngrams(tokens, ngram_size)
|
||||
|
||||
fingerprints = set()
|
||||
for i, ng in enumerate(ngrams):
|
||||
if i % sample_rate == 0:
|
||||
h = hashlib.md5("".join(ng).encode(), usedforsecurity=False).hexdigest()[:8]
|
||||
fingerprints.add(h)
|
||||
|
||||
return fingerprints
|
||||
|
||||
|
||||
def compare_documents(fp1: set[str], fp2: set[str]) -> float:
|
||||
"""Jaccard similarity between two fingerprints."""
|
||||
if not fp1 or not fp2:
|
||||
return 0.0
|
||||
intersection = len(fp1 & fp2)
|
||||
union = len(fp1 | fp2)
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def read_document(filepath: str) -> str:
|
||||
"""Read a document's raw text content."""
|
||||
with Path(filepath).open(encoding="utf-8", errors="ignore") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Fingerprint every input file and report pairwise similarity."""
|
||||
parser = argparse.ArgumentParser(description="Sherlock - Text Plagiarism Detector")
|
||||
parser.add_argument("files", nargs="+", help="Files to compare")
|
||||
parser.add_argument(
|
||||
"--ngram", "-n", type=int, default=3, help="N-gram size (default: 3)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--threshold", "-t", type=float, default=0.1, help="Similarity threshold"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(args.files) < MIN_FILES:
|
||||
_out("Need at least 2 files to compare")
|
||||
return 1
|
||||
|
||||
docs = {}
|
||||
for f in args.files:
|
||||
if Path(f).exists():
|
||||
text = read_document(f)
|
||||
docs[f] = fingerprint(text, args.ngram)
|
||||
|
||||
_out("\nSherlock Plagiarism Analysis")
|
||||
_out("=" * 50)
|
||||
|
||||
files = list(docs.keys())
|
||||
for file1, file2 in combinations(files, 2):
|
||||
sim = compare_documents(docs[file1], docs[file2])
|
||||
name1 = Path(file1).name
|
||||
name2 = Path(file2).name
|
||||
flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else ""
|
||||
_out(f"{name1} <-> {name2}: {sim:.1%}{flag}")
|
||||
|
||||
_out("=" * 50)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Reference in New Issue
Block a user