mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-11 22:14:01 +01:00
Krzysztof kuhy Rudnicki
1b19808edb
analyze_repo.sh was shfmt-clean at133fccfand my split made it dirty (a double blank line). Fixed with shfmt -w, then restored the STEP 1 comment banner shfmt -w also stripped -- all eight banners re-counted. The three new shell_check libs inherited their parent's 2-space indentation; as new files they have no prior style to preserve, so they are now tabs like shfmt wants. Context for the exit code: 157 of 298 linux_configuration shell files already fail shfmt at133fccf. shfmt is not in .pre-commit-config.yaml and there is no .editorconfig, so run_linters exiting 1 on the full tree is the repo's standing state, not a regression here. Scoped this commit to what these splits touched rather than reformatting 157 files. Also closes the risk left open ina84556a: run_linters now verified directly over the three new libs -- all six passes PASS, exit 0 -- with install_linters never invoked, so decision 6 holds. That is the path .github/workflows/shell-check.yml runs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
235 lines
7.8 KiB
Bash
Executable File
235 lines
7.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# Analyze a git repository for most-used keywords, functions, etc.
|
|
# Usage: ./analyze_repo.sh [repo_url_or_local_path] [output_dir] [--no-ignore]
|
|
#
|
|
# Examples:
|
|
# ./analyze_repo.sh https://github.com/torvalds/linux # Clone from URL
|
|
# ./analyze_repo.sh /path/to/local/repo # Use local directory
|
|
# ./analyze_repo.sh . # Analyze current directory
|
|
# ./analyze_repo.sh . /tmp/out --no-ignore # Include node_modules, etc.
|
|
|
|
set -e
|
|
|
|
# Resolve this script's directory up front (absolute), before any cd, so sibling
|
|
# helpers like fast_count.py remain locatable once we cd into the analyzed repo.
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# Parse arguments
|
|
INPUT=""
|
|
WORK_DIR=""
|
|
RESPECT_GITIGNORE=true
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--no-ignore)
|
|
RESPECT_GITIGNORE=false
|
|
;;
|
|
*)
|
|
if [ -z "$INPUT" ]; then
|
|
INPUT="$arg"
|
|
elif [ -z "$WORK_DIR" ]; then
|
|
WORK_DIR="$arg"
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
|
|
INPUT="${INPUT:-https://github.com/torvalds/linux}"
|
|
WORK_DIR="${WORK_DIR:-/tmp/repo_analysis}"
|
|
TOP_N=50 # Number of top results to show
|
|
|
|
# Directories to exclude (unless --no-ignore is used)
|
|
EXCLUDE_DIRS="node_modules|\.git|vendor|\.venv|venv|__pycache__|\.cache|build|dist|\.next|\.nuxt|target|\.tox|\.eggs"
|
|
|
|
# Detect if input is a URL or local path
|
|
is_url() {
|
|
[[ $1 =~ ^https?:// ]] || [[ $1 =~ ^git@ ]] || [[ $1 =~ ^ssh:// ]]
|
|
}
|
|
|
|
IS_LOCAL=false
|
|
if is_url "$INPUT"; then
|
|
REPO_URL="$INPUT"
|
|
REPO_NAME=$(basename "$REPO_URL" .git)
|
|
REPO_DIR="$WORK_DIR/$REPO_NAME"
|
|
else
|
|
# Local path - resolve to absolute path
|
|
IS_LOCAL=true
|
|
if [ -d "$INPUT" ]; then
|
|
REPO_DIR=$(cd "$INPUT" && pwd)
|
|
REPO_NAME=$(basename "$REPO_DIR")
|
|
else
|
|
echo "Error: '$INPUT' is not a valid directory or URL"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
RESULTS_DIR="$WORK_DIR/results_${REPO_NAME}"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
BLUE='\033[0;34m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
#==============================================================================
|
|
# Libraries
|
|
#==============================================================================
|
|
# Sourced after the colour constants and configuration above, which they read.
|
|
source "$SCRIPT_DIR/lib/analyze_repo_scan.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_tools.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_separate.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_report.sh"
|
|
source "$SCRIPT_DIR/lib/analyze_repo_symbols.sh"
|
|
|
|
#==============================================================================
|
|
# STEP 0: Install Missing Tools
|
|
#==============================================================================
|
|
print_header "STEP 0: Checking/Installing Required Tools"
|
|
install_missing_tools
|
|
|
|
# Create directories
|
|
mkdir -p "$WORK_DIR" "$RESULTS_DIR"
|
|
|
|
#==============================================================================
|
|
# STEP 1: Clone or Use Local Repository
|
|
#==============================================================================
|
|
print_header "STEP 1: Repository Setup"
|
|
|
|
if [ "$IS_LOCAL" = true ]; then
|
|
echo "Using local repository: $REPO_DIR"
|
|
if [ ! -d "$REPO_DIR" ]; then
|
|
echo "Error: Directory does not exist: $REPO_DIR"
|
|
exit 1
|
|
fi
|
|
else
|
|
# Remote URL - clone it
|
|
if [ -d "$REPO_DIR" ]; then
|
|
echo "Repository already exists at $REPO_DIR"
|
|
echo "Updating..."
|
|
cd "$REPO_DIR"
|
|
git pull --depth 1 2>/dev/null || echo "Update skipped (shallow clone)"
|
|
else
|
|
echo "Cloning $REPO_URL (shallow clone for speed)..."
|
|
git clone --depth 1 "$REPO_URL" "$REPO_DIR"
|
|
fi
|
|
fi
|
|
|
|
cd "$REPO_DIR"
|
|
echo "Repository: $REPO_NAME"
|
|
echo "Location: $REPO_DIR"
|
|
echo "Repository size: $(du -sh . | cut -f1)"
|
|
if [ "$RESPECT_GITIGNORE" = true ] && is_git_repo; then
|
|
# Count files respecting .gitignore
|
|
FILE_COUNT=$({
|
|
git ls-files 2>/dev/null
|
|
git ls-files --others --exclude-standard 2>/dev/null
|
|
} | sort -u | wc -l)
|
|
echo "Files: $FILE_COUNT (respecting .gitignore)"
|
|
elif [ "$RESPECT_GITIGNORE" = true ]; then
|
|
echo "Files: $(find . -type f 2>/dev/null | grep -cEv "/($EXCLUDE_DIRS)/") (excluding common dirs)"
|
|
else
|
|
echo "Files: $(find . -type f | wc -l)"
|
|
fi
|
|
|
|
#==============================================================================
|
|
# STEP 2: Basic Statistics with tokei
|
|
#==============================================================================
|
|
print_header "STEP 2: Code Statistics with tokei"
|
|
|
|
echo "Running tokei..."
|
|
tokei . | tee "$RESULTS_DIR/tokei_stats.txt"
|
|
|
|
#==============================================================================
|
|
# STEP 3: Code Statistics with scc
|
|
#==============================================================================
|
|
print_header "STEP 3: Code Statistics with scc (includes complexity)"
|
|
|
|
echo "Running scc..."
|
|
scc . | tee "$RESULTS_DIR/scc_stats.txt"
|
|
|
|
print_subheader "Top 10 Most Complex Files"
|
|
scc --by-file --sort complexity . 2>/dev/null | head -20 | tee "$RESULTS_DIR/scc_complexity.txt"
|
|
|
|
#==============================================================================
|
|
# STEP 4: Fast Keyword Analysis (Code vs Comments) - Multi-Language
|
|
#==============================================================================
|
|
print_header "STEP 4: Fast Keyword Analysis (Code vs Comments)"
|
|
|
|
detect_languages
|
|
|
|
#------------------------------------------------------------------------------
|
|
# Multi-language comment processing - KEEP LANGUAGES SEPARATE
|
|
#------------------------------------------------------------------------------
|
|
print_subheader "Processing source files (separating code from comments)..."
|
|
|
|
COMMENTS_TEMP=$(mktemp)
|
|
trap 'rm -f "$COMMENTS_TEMP" /tmp/code_*.tmp 2>/dev/null' EXIT
|
|
# Named once and passed by name to both passes, so the map itself never becomes
|
|
# a global that the libs reach for implicitly.
|
|
CODE_FILES_MAP="LANG_CODE_FILES"
|
|
declare -A "$CODE_FILES_MAP"
|
|
|
|
separate_code_and_comments "$CODE_FILES_MAP" "$COMMENTS_TEMP"
|
|
report_languages "$CODE_FILES_MAP" "$COMMENTS_TEMP"
|
|
|
|
#==============================================================================
|
|
# STEP 5: ctags Symbol Analysis
|
|
#==============================================================================
|
|
analyze_symbols
|
|
|
|
#==============================================================================
|
|
# STEP 6: cscope Analysis
|
|
#==============================================================================
|
|
print_header "STEP 6: cscope Database Analysis"
|
|
analyze_cscope
|
|
|
|
#==============================================================================
|
|
# STEP 7: clang AST Analysis (if available)
|
|
#==============================================================================
|
|
print_header "STEP 7: clang-based Analysis (AST-level)"
|
|
analyze_clang_ast
|
|
|
|
#==============================================================================
|
|
# STEP 8: Summary
|
|
#==============================================================================
|
|
print_header "ANALYSIS COMPLETE"
|
|
|
|
echo "Results saved to: $RESULTS_DIR/"
|
|
echo ""
|
|
ls -la "$RESULTS_DIR/"
|
|
|
|
echo ""
|
|
echo -e "${GREEN}Quick Summary:${NC}"
|
|
echo ""
|
|
|
|
if [ -f "$RESULTS_DIR/grep_keywords.txt" ]; then
|
|
echo "Top 5 Language Keywords (in code):"
|
|
head -5 "$RESULTS_DIR/grep_keywords.txt" | awk '{printf " %s: %s times\n", $2, $1}'
|
|
fi
|
|
|
|
echo ""
|
|
if [ -f "$RESULTS_DIR/grep_function_calls.txt" ]; then
|
|
echo "Top 5 Function/Method Calls (in code):"
|
|
head -5 "$RESULTS_DIR/grep_function_calls.txt" | awk '{printf " %s(): %s times\n", $2, $1}'
|
|
fi
|
|
|
|
echo ""
|
|
if [ -f "$RESULTS_DIR/comment_words.txt" ]; then
|
|
echo "Top 5 Words in Comments:"
|
|
head -5 "$RESULTS_DIR/comment_words.txt" | awk '{printf " %s: %s times\n", $2, $1}'
|
|
fi
|
|
|
|
echo ""
|
|
if [ -f "$RESULTS_DIR/grep_imports.txt" ]; then
|
|
echo "Top 5 Imports/Includes:"
|
|
head -5 "$RESULTS_DIR/grep_imports.txt" | awk '{count=$1; $1=""; printf " %s: %s times\n", substr($0,2), count}'
|
|
fi
|
|
|
|
echo ""
|
|
echo -e "${BLUE}To explore interactively with cscope (C/C++ only):${NC}"
|
|
echo " cd $REPO_DIR && cscope -d -f $RESULTS_DIR/cscope.out"
|
|
echo ""
|
|
echo -e "${BLUE}To browse tags in vim:${NC}"
|
|
echo " cd $REPO_DIR && vim -t main"
|