mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-12 16:30:58 +01:00
Krzysztof kuhy Rudnicki
41cc0451c9
869 lines -> 231, with the extracted helpers and per-STEP blocks in
lib/analyze_repo_{scan,tools,separate,report,symbols}.sh (86-228 each).
The seam that mattered was STEP 4, whose two passes share a map of
language -> temp file. Rather than let that cross a file boundary, the
map is passed by nameref and the comments file by parameter, so
separate_code_and_comments fills what report_languages reads with no
global in between.
Two boundaries were one step off on the first pass and produced 23
shellcheck findings: the KEYWORDS_* tables and detect_languages had been
cut away from their only consumers, so every one read as unused (SC2034)
or unassigned (SC2154). Moving each beside its consumer cleared all 23
without a suppression. The nameref also makes shellcheck read a bare
assoc-array subscript on the left of an assignment as arithmetic;
quoting the nine keys is the fix.
The line-set diff against HEAD earned its place here: it caught a
`find ... -printf` call at original line 719 that had fallen into a
one-line gap between two extraction ranges and been dropped. bash -n and
shellcheck both passed on that broken version.
Verified without running the script (decision 6 -- install_missing_tools
calls sudo pacman, cargo and makepkg): shellcheck clean across all six,
the function set matches HEAD plus exactly the six new wrappers, no
executable line lost, and a stubbed run of the entry preamble defines
all fourteen functions, proving the source lines resolve at runtime.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
232 lines
7.6 KiB
Bash
Executable File
232 lines
7.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# Analyze a git repository for most-used keywords, functions, etc.
|
|
# Usage: ./analyze_repo.sh [repo_url_or_local_path] [output_dir] [--no-ignore]
|
|
#
|
|
# Examples:
|
|
# ./analyze_repo.sh https://github.com/torvalds/linux # Clone from URL
|
|
# ./analyze_repo.sh /path/to/local/repo # Use local directory
|
|
# ./analyze_repo.sh . # Analyze current directory
|
|
# ./analyze_repo.sh . /tmp/out --no-ignore # Include node_modules, etc.
|
|
|
|
set -e
|
|
|
|
# Resolve this script's directory up front (absolute), before any cd, so sibling
|
|
# helpers like fast_count.py remain locatable once we cd into the analyzed repo.
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# Parse arguments
|
|
INPUT=""
|
|
WORK_DIR=""
|
|
RESPECT_GITIGNORE=true
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--no-ignore)
|
|
RESPECT_GITIGNORE=false
|
|
;;
|
|
*)
|
|
if [ -z "$INPUT" ]; then
|
|
INPUT="$arg"
|
|
elif [ -z "$WORK_DIR" ]; then
|
|
WORK_DIR="$arg"
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
|
|
INPUT="${INPUT:-https://github.com/torvalds/linux}"
|
|
WORK_DIR="${WORK_DIR:-/tmp/repo_analysis}"
|
|
TOP_N=50 # Number of top results to show
|
|
|
|
# Directories to exclude (unless --no-ignore is used)
|
|
EXCLUDE_DIRS="node_modules|\.git|vendor|\.venv|venv|__pycache__|\.cache|build|dist|\.next|\.nuxt|target|\.tox|\.eggs"
|
|
|
|
# Detect if input is a URL or local path
|
|
is_url() {
|
|
[[ $1 =~ ^https?:// ]] || [[ $1 =~ ^git@ ]] || [[ $1 =~ ^ssh:// ]]
|
|
}
|
|
|
|
IS_LOCAL=false
|
|
if is_url "$INPUT"; then
|
|
REPO_URL="$INPUT"
|
|
REPO_NAME=$(basename "$REPO_URL" .git)
|
|
REPO_DIR="$WORK_DIR/$REPO_NAME"
|
|
else
|
|
# Local path - resolve to absolute path
|
|
IS_LOCAL=true
|
|
if [ -d "$INPUT" ]; then
|
|
REPO_DIR=$(cd "$INPUT" && pwd)
|
|
REPO_NAME=$(basename "$REPO_DIR")
|
|
else
|
|
echo "Error: '$INPUT' is not a valid directory or URL"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
RESULTS_DIR="$WORK_DIR/results_${REPO_NAME}"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
BLUE='\033[0;34m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
|
|
#==============================================================================
|
|
# Libraries
|
|
#==============================================================================
|
|
# Sourced after the colour constants and configuration above, which they read.
|
|
source "$SCRIPT_DIR/lib/analyze_repo_scan.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_tools.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_separate.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_report.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_symbols.sh"
|
|
|
|
#==============================================================================
|
|
# STEP 0: Install Missing Tools
|
|
#==============================================================================
|
|
print_header "STEP 0: Checking/Installing Required Tools"
|
|
install_missing_tools
|
|
|
|
# Create directories
|
|
mkdir -p "$WORK_DIR" "$RESULTS_DIR"
|
|
|
|
# STEP 1: Clone or Use Local Repository
|
|
#==============================================================================
|
|
print_header "STEP 1: Repository Setup"
|
|
|
|
if [ "$IS_LOCAL" = true ]; then
|
|
echo "Using local repository: $REPO_DIR"
|
|
if [ ! -d "$REPO_DIR" ]; then
|
|
echo "Error: Directory does not exist: $REPO_DIR"
|
|
exit 1
|
|
fi
|
|
else
|
|
# Remote URL - clone it
|
|
if [ -d "$REPO_DIR" ]; then
|
|
echo "Repository already exists at $REPO_DIR"
|
|
echo "Updating..."
|
|
cd "$REPO_DIR"
|
|
git pull --depth 1 2>/dev/null || echo "Update skipped (shallow clone)"
|
|
else
|
|
echo "Cloning $REPO_URL (shallow clone for speed)..."
|
|
git clone --depth 1 "$REPO_URL" "$REPO_DIR"
|
|
fi
|
|
fi
|
|
|
|
cd "$REPO_DIR"
|
|
echo "Repository: $REPO_NAME"
|
|
echo "Location: $REPO_DIR"
|
|
echo "Repository size: $(du -sh . | cut -f1)"
|
|
if [ "$RESPECT_GITIGNORE" = true ] && is_git_repo; then
|
|
# Count files respecting .gitignore
|
|
FILE_COUNT=$({
|
|
git ls-files 2>/dev/null
|
|
git ls-files --others --exclude-standard 2>/dev/null
|
|
} | sort -u | wc -l)
|
|
echo "Files: $FILE_COUNT (respecting .gitignore)"
|
|
elif [ "$RESPECT_GITIGNORE" = true ]; then
|
|
echo "Files: $(find . -type f 2>/dev/null | grep -cEv "/($EXCLUDE_DIRS)/") (excluding common dirs)"
|
|
else
|
|
echo "Files: $(find . -type f | wc -l)"
|
|
fi
|
|
|
|
#==============================================================================
|
|
# STEP 2: Basic Statistics with tokei
|
|
#==============================================================================
|
|
print_header "STEP 2: Code Statistics with tokei"
|
|
|
|
echo "Running tokei..."
|
|
tokei . | tee "$RESULTS_DIR/tokei_stats.txt"
|
|
|
|
#==============================================================================
|
|
# STEP 3: Code Statistics with scc
|
|
#==============================================================================
|
|
print_header "STEP 3: Code Statistics with scc (includes complexity)"
|
|
|
|
echo "Running scc..."
|
|
scc . | tee "$RESULTS_DIR/scc_stats.txt"
|
|
|
|
print_subheader "Top 10 Most Complex Files"
|
|
scc --by-file --sort complexity . 2>/dev/null | head -20 | tee "$RESULTS_DIR/scc_complexity.txt"
|
|
|
|
#==============================================================================
|
|
# STEP 4: Fast Keyword Analysis (Code vs Comments) - Multi-Language
|
|
#==============================================================================
|
|
print_header "STEP 4: Fast Keyword Analysis (Code vs Comments)"
|
|
|
|
detect_languages
|
|
|
|
#------------------------------------------------------------------------------
|
|
# Multi-language comment processing - KEEP LANGUAGES SEPARATE
|
|
#------------------------------------------------------------------------------
|
|
print_subheader "Processing source files (separating code from comments)..."
|
|
|
|
COMMENTS_TEMP=$(mktemp)
|
|
trap 'rm -f "$COMMENTS_TEMP" /tmp/code_*.tmp 2>/dev/null' EXIT
|
|
declare -A LANG_CODE_FILES
|
|
|
|
separate_code_and_comments LANG_CODE_FILES "$COMMENTS_TEMP"
|
|
report_languages LANG_CODE_FILES "$COMMENTS_TEMP"
|
|
|
|
#==============================================================================
|
|
# STEP 5: ctags Symbol Analysis
|
|
#==============================================================================
|
|
analyze_symbols
|
|
|
|
#==============================================================================
|
|
# STEP 6: cscope Analysis
|
|
#==============================================================================
|
|
print_header "STEP 6: cscope Database Analysis"
|
|
analyze_cscope
|
|
|
|
#==============================================================================
|
|
# STEP 7: clang AST Analysis (if available)
|
|
#==============================================================================
|
|
print_header "STEP 7: clang-based Analysis (AST-level)"
|
|
analyze_clang_ast
|
|
|
|
#==============================================================================
|
|
# STEP 8: Summary
|
|
#==============================================================================
|
|
print_header "ANALYSIS COMPLETE"
|
|
|
|
echo "Results saved to: $RESULTS_DIR/"
|
|
echo ""
|
|
ls -la "$RESULTS_DIR/"
|
|
|
|
echo ""
|
|
echo -e "${GREEN}Quick Summary:${NC}"
|
|
echo ""
|
|
|
|
if [ -f "$RESULTS_DIR/grep_keywords.txt" ]; then
|
|
echo "Top 5 Language Keywords (in code):"
|
|
head -5 "$RESULTS_DIR/grep_keywords.txt" | awk '{printf " %s: %s times\n", $2, $1}'
|
|
fi
|
|
|
|
echo ""
|
|
if [ -f "$RESULTS_DIR/grep_function_calls.txt" ]; then
|
|
echo "Top 5 Function/Method Calls (in code):"
|
|
head -5 "$RESULTS_DIR/grep_function_calls.txt" | awk '{printf " %s(): %s times\n", $2, $1}'
|
|
fi
|
|
|
|
echo ""
|
|
if [ -f "$RESULTS_DIR/comment_words.txt" ]; then
|
|
echo "Top 5 Words in Comments:"
|
|
head -5 "$RESULTS_DIR/comment_words.txt" | awk '{printf " %s: %s times\n", $2, $1}'
|
|
fi
|
|
|
|
echo ""
|
|
if [ -f "$RESULTS_DIR/grep_imports.txt" ]; then
|
|
echo "Top 5 Imports/Includes:"
|
|
head -5 "$RESULTS_DIR/grep_imports.txt" | awk '{count=$1; $1=""; printf " %s: %s times\n", substr($0,2), count}'
|
|
fi
|
|
|
|
echo ""
|
|
echo -e "${BLUE}To explore interactively with cscope (C/C++ only):${NC}"
|
|
echo " cd $REPO_DIR && cscope -d -f $RESULTS_DIR/cscope.out"
|
|
echo ""
|
|
echo -e "${BLUE}To browse tags in vim:${NC}"
|
|
echo " cd $REPO_DIR && vim -t main"
|