mirror of
https://github.com/kuhyx/testsAndMisc-archive.git
synced 2026-11-10 22:47:05 +01:00
Krzysztof kuhy Rudnicki
e47c72329a
Wraps the four install stages into named functions, one per lib/plagiarism_*.sh sibling (matching install_exercism.sh's convention), and extracts the two embedded Python heredocs into real, lint-clean check_plagiarism.py/sherlock.py files -- check_plagiarism.py needed a second split into plagiarism_similarity.py to also clear the 250-line cap. Every function reaches 100% shell coverage via a PATH-shim test harness (fake python3/pip/git/java/wine); the two Python CLIs were separately verified against a real venv with actual nltk/scikit-learn. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
109 lines
3.3 KiB
Python
Executable File
109 lines
3.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Sherlock - simple text plagiarism detector using n-gram fingerprinting.
|
|
|
|
Extracted from `install_plagiarism_tools.sh`'s fallback path, which used to
|
|
write this out via a `cat <<'SHERLOCKEOF'` heredoc when cloning the real
|
|
sherlock-py failed. Inline heredoc Python is invisible to ruff, mypy, pylint,
|
|
bandit and pytest, so it lives in its own module instead, matching
|
|
`download_nltk_data.py`'s extraction.
|
|
|
|
The installer copies this file into `$SHERLOCK_DIR/sherlock.py` only when
|
|
`git clone` of the upstream tool fails.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
from itertools import combinations
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
#: Minimum number of input files for a pairwise comparison to make sense.
|
|
MIN_FILES = 2
|
|
|
|
|
|
def _out(line: str = "") -> None:
|
|
sys.stdout.write(line + "\n")
|
|
|
|
|
|
def tokenize(text: str) -> list[str]:
|
|
"""Split text into lowercase alphanumeric words."""
|
|
return [w.lower() for w in text.split() if w.isalnum()]
|
|
|
|
|
|
def get_ngrams(tokens: list[str], n: int = 3) -> list[tuple[str, ...]]:
|
|
"""Generate n-grams from a token list."""
|
|
return [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)]
|
|
|
|
|
|
def fingerprint(text: str, ngram_size: int = 3, sample_rate: int = 4) -> set[str]:
|
|
"""Create a document fingerprint from sampled n-gram hashes."""
|
|
tokens = tokenize(text)
|
|
ngrams = get_ngrams(tokens, ngram_size)
|
|
|
|
fingerprints = set()
|
|
for i, ng in enumerate(ngrams):
|
|
if i % sample_rate == 0:
|
|
h = hashlib.md5("".join(ng).encode(), usedforsecurity=False).hexdigest()[:8]
|
|
fingerprints.add(h)
|
|
|
|
return fingerprints
|
|
|
|
|
|
def compare_documents(fp1: set[str], fp2: set[str]) -> float:
|
|
"""Jaccard similarity between two fingerprints."""
|
|
if not fp1 or not fp2:
|
|
return 0.0
|
|
intersection = len(fp1 & fp2)
|
|
union = len(fp1 | fp2)
|
|
return intersection / union if union > 0 else 0.0
|
|
|
|
|
|
def read_document(filepath: str) -> str:
|
|
"""Read a document's raw text content."""
|
|
with Path(filepath).open(encoding="utf-8", errors="ignore") as f:
|
|
return f.read()
|
|
|
|
|
|
def main() -> int:
|
|
"""Fingerprint every input file and report pairwise similarity."""
|
|
parser = argparse.ArgumentParser(description="Sherlock - Text Plagiarism Detector")
|
|
parser.add_argument("files", nargs="+", help="Files to compare")
|
|
parser.add_argument(
|
|
"--ngram", "-n", type=int, default=3, help="N-gram size (default: 3)"
|
|
)
|
|
parser.add_argument(
|
|
"--threshold", "-t", type=float, default=0.1, help="Similarity threshold"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if len(args.files) < MIN_FILES:
|
|
_out("Need at least 2 files to compare")
|
|
return 1
|
|
|
|
docs = {}
|
|
for f in args.files:
|
|
if Path(f).exists():
|
|
text = read_document(f)
|
|
docs[f] = fingerprint(text, args.ngram)
|
|
|
|
_out("\nSherlock Plagiarism Analysis")
|
|
_out("=" * 50)
|
|
|
|
files = list(docs.keys())
|
|
for file1, file2 in combinations(files, 2):
|
|
sim = compare_documents(docs[file1], docs[file2])
|
|
name1 = Path(file1).name
|
|
name2 = Path(file2).name
|
|
flag = " ⚠️ SUSPICIOUS" if sim >= args.threshold else ""
|
|
_out(f"{name1} <-> {name2}: {sim:.1%}{flag}")
|
|
|
|
_out("=" * 50)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|