testsAndMisc/linux_configuration/utils/analyze_repo.sh

#!/bin/bash
# Analyze a git repository for most-used keywords, functions, etc.
# Usage: ./analyze_repo.sh [repo_url_or_local_path] [output_dir] [--no-ignore]
#
# Examples:
#   ./analyze_repo.sh https://github.com/torvalds/linux    # Clone from URL
#   ./analyze_repo.sh /path/to/local/repo                  # Use local directory
#   ./analyze_repo.sh .                                    # Analyze current directory
#   ./analyze_repo.sh . /tmp/out --no-ignore               # Include node_modules, etc.

set -e

# Resolve this script's directory up front (absolute), before any cd, so sibling
# helpers like fast_count.py remain locatable once we cd into the analyzed repo.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# Parse arguments
INPUT=""
WORK_DIR=""
RESPECT_GITIGNORE=true

for arg in "$@"; do
	case "$arg" in
	--no-ignore)
		RESPECT_GITIGNORE=false
		;;
	*)
		if [ -z "$INPUT" ]; then
			INPUT="$arg"
		elif [ -z "$WORK_DIR" ]; then
			WORK_DIR="$arg"
		fi
		;;
	esac
done

INPUT="${INPUT:-https://github.com/torvalds/linux}"
WORK_DIR="${WORK_DIR:-/tmp/repo_analysis}"
TOP_N=50 # Number of top results to show

# Directories to exclude (unless --no-ignore is used)
EXCLUDE_DIRS="node_modules|\.git|vendor|\.venv|venv|__pycache__|\.cache|build|dist|\.next|\.nuxt|target|\.tox|\.eggs"

# Detect if input is a URL or local path
is_url() {
	[[ $1 =~ ^https?:// ]] || [[ $1 =~ ^git@ ]] || [[ $1 =~ ^ssh:// ]]
}

IS_LOCAL=false
if is_url "$INPUT"; then
	REPO_URL="$INPUT"
	REPO_NAME=$(basename "$REPO_URL" .git)
	REPO_DIR="$WORK_DIR/$REPO_NAME"
else
	# Local path - resolve to absolute path
	IS_LOCAL=true
	if [ -d "$INPUT" ]; then
		REPO_DIR=$(cd "$INPUT" && pwd)
		REPO_NAME=$(basename "$REPO_DIR")
	else
		echo "Error: '$INPUT' is not a valid directory or URL"
		exit 1
	fi
fi

RESULTS_DIR="$WORK_DIR/results_${REPO_NAME}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

#==============================================================================
# Libraries
#==============================================================================
# Sourced after the colour constants and configuration above, which they read.
source "$SCRIPT_DIR/lib/analyze_repo_scan.sh"
source "$SCRIPT_DIR/lib/analyze_repo_tools.sh"
source "$SCRIPT_DIR/lib/analyze_repo_separate.sh"
source "$SCRIPT_DIR/lib/analyze_repo_report.sh"
source "$SCRIPT_DIR/lib/analyze_repo_symbols.sh"

#==============================================================================
# STEP 0: Install Missing Tools
#==============================================================================
print_header "STEP 0: Checking/Installing Required Tools"
install_missing_tools

# Create directories
mkdir -p "$WORK_DIR" "$RESULTS_DIR"

#==============================================================================
# STEP 1: Clone or Use Local Repository
#==============================================================================
print_header "STEP 1: Repository Setup"

if [ "$IS_LOCAL" = true ]; then
	echo "Using local repository: $REPO_DIR"
	if [ ! -d "$REPO_DIR" ]; then
		echo "Error: Directory does not exist: $REPO_DIR"
		exit 1
	fi
else
	# Remote URL - clone it
	if [ -d "$REPO_DIR" ]; then
		echo "Repository already exists at $REPO_DIR"
		echo "Updating..."
		cd "$REPO_DIR"
		git pull --depth 1 2>/dev/null || echo "Update skipped (shallow clone)"
	else
		echo "Cloning $REPO_URL (shallow clone for speed)..."
		git clone --depth 1 "$REPO_URL" "$REPO_DIR"
	fi
fi

cd "$REPO_DIR"
echo "Repository: $REPO_NAME"
echo "Location: $REPO_DIR"
echo "Repository size: $(du -sh . | cut -f1)"
if [ "$RESPECT_GITIGNORE" = true ] && is_git_repo; then
	# Count files respecting .gitignore
	FILE_COUNT=$({
		git ls-files 2>/dev/null
		git ls-files --others --exclude-standard 2>/dev/null
	} | sort -u | wc -l)
	echo "Files: $FILE_COUNT (respecting .gitignore)"
elif [ "$RESPECT_GITIGNORE" = true ]; then
	echo "Files: $(find . -type f 2>/dev/null | grep -cEv "/($EXCLUDE_DIRS)/") (excluding common dirs)"
else
	echo "Files: $(find . -type f | wc -l)"
fi

#==============================================================================
# STEP 2: Basic Statistics with tokei
#==============================================================================
print_header "STEP 2: Code Statistics with tokei"

echo "Running tokei..."
tokei . | tee "$RESULTS_DIR/tokei_stats.txt"

#==============================================================================
# STEP 3: Code Statistics with scc
#==============================================================================
print_header "STEP 3: Code Statistics with scc (includes complexity)"

echo "Running scc..."
scc . | tee "$RESULTS_DIR/scc_stats.txt"

print_subheader "Top 10 Most Complex Files"
scc --by-file --sort complexity . 2>/dev/null | head -20 | tee "$RESULTS_DIR/scc_complexity.txt"

#==============================================================================
# STEP 4: Fast Keyword Analysis (Code vs Comments) - Multi-Language
#==============================================================================
print_header "STEP 4: Fast Keyword Analysis (Code vs Comments)"

detect_languages

#------------------------------------------------------------------------------
# Multi-language comment processing - KEEP LANGUAGES SEPARATE
#------------------------------------------------------------------------------
print_subheader "Processing source files (separating code from comments)..."

COMMENTS_TEMP=$(mktemp)
trap 'rm -f "$COMMENTS_TEMP" /tmp/code_*.tmp 2>/dev/null' EXIT
# Named once and passed by name to both passes, so the map itself never becomes
# a global that the libs reach for implicitly.
CODE_FILES_MAP="LANG_CODE_FILES"
declare -A "$CODE_FILES_MAP"

separate_code_and_comments "$CODE_FILES_MAP" "$COMMENTS_TEMP"
report_languages "$CODE_FILES_MAP" "$COMMENTS_TEMP"

#==============================================================================
# STEP 5: ctags Symbol Analysis
#==============================================================================
analyze_symbols

#==============================================================================
# STEP 6: cscope Analysis
#==============================================================================
print_header "STEP 6: cscope Database Analysis"
analyze_cscope

#==============================================================================
# STEP 7: clang AST Analysis (if available)
#==============================================================================
print_header "STEP 7: clang-based Analysis (AST-level)"
analyze_clang_ast

#==============================================================================
# STEP 8: Summary
#==============================================================================
print_header "ANALYSIS COMPLETE"

echo "Results saved to: $RESULTS_DIR/"
echo ""
ls -la "$RESULTS_DIR/"

echo ""
echo -e "${GREEN}Quick Summary:${NC}"
echo ""

if [ -f "$RESULTS_DIR/grep_keywords.txt" ]; then
	echo "Top 5 Language Keywords (in code):"
	head -5 "$RESULTS_DIR/grep_keywords.txt" | awk '{printf "  %s: %s times\n", $2, $1}'
fi

echo ""
if [ -f "$RESULTS_DIR/grep_function_calls.txt" ]; then
	echo "Top 5 Function/Method Calls (in code):"
	head -5 "$RESULTS_DIR/grep_function_calls.txt" | awk '{printf "  %s(): %s times\n", $2, $1}'
fi

echo ""
if [ -f "$RESULTS_DIR/comment_words.txt" ]; then
	echo "Top 5 Words in Comments:"
	head -5 "$RESULTS_DIR/comment_words.txt" | awk '{printf "  %s: %s times\n", $2, $1}'
fi

echo ""
if [ -f "$RESULTS_DIR/grep_imports.txt" ]; then
	echo "Top 5 Imports/Includes:"
	head -5 "$RESULTS_DIR/grep_imports.txt" | awk '{count=$1; $1=""; printf "  %s: %s times\n", substr($0,2), count}'
fi

echo ""
echo -e "${BLUE}To explore interactively with cscope (C/C++ only):${NC}"
echo "  cd $REPO_DIR && cscope -d -f $RESULTS_DIR/cscope.out"
echo ""
echo -e "${BLUE}To browse tags in vim:${NC}"
echo "  cd $REPO_DIR && vim -t main"