From 41cc0451c9db4ef43dfb42dd3d8a019eebfbbd90 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 16 Aug 2026 21:47:03 +0200 Subject: [PATCH] Split analyze_repo.sh into an entry script and five libs 869 lines -> 231, with the extracted helpers and per-STEP blocks in lib/analyze_repo_{scan,tools,separate,report,symbols}.sh (86-228 each). The seam that mattered was STEP 4, whose two passes share a map of language -> temp file. Rather than let that cross a file boundary, the map is passed by nameref and the comments file by parameter, so separate_code_and_comments fills what report_languages reads with no global in between. Two boundaries were one step off on the first pass and produced 23 shellcheck findings: the KEYWORDS_* tables and detect_languages had been cut away from their only consumers, so every one read as unused (SC2034) or unassigned (SC2154). Moving each beside its consumer cleared all 23 without a suppression. The nameref also makes shellcheck read a bare assoc-array subscript on the left of an assignment as arithmetic; quoting the nine keys is the fix. The line-set diff against HEAD earned its place here: it caught a `find ... -printf` call at original line 719 that had fallen into a one-line gap between two extraction ranges and been dropped. bash -n and shellcheck both passed on that broken version. Verified without running the script (decision 6 -- install_missing_tools calls sudo pacman, cargo and makepkg): shellcheck clean across all six, the function set matches HEAD plus exactly the six new wrappers, no executable line lost, and a stubbed run of the entry preamble defines all fourteen functions, proving the source lines resolve at runtime. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY --- .../analyze-repo-split-2026-08-16.json | 19 + .../analyze-repo-split-2026-08-16.json | 64 ++ .../scripts/single_use/utils/analyze_repo.sh | 674 +----------------- .../utils/lib/analyze_repo_report.sh | 228 ++++++ .../single_use/utils/lib/analyze_repo_scan.sh | 86 +++ .../utils/lib/analyze_repo_separate.sh | 186 +++++ .../utils/lib/analyze_repo_symbols.sh | 112 +++ .../utils/lib/analyze_repo_tools.sh | 124 ++++ 8 files changed, 837 insertions(+), 656 deletions(-) create mode 100644 docs/superpowers/contracts/analyze-repo-split-2026-08-16.json create mode 100644 docs/superpowers/evidence/analyze-repo-split-2026-08-16.json create mode 100755 linux_configuration/scripts/single_use/utils/lib/analyze_repo_report.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/analyze_repo_scan.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/analyze_repo_separate.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/analyze_repo_symbols.sh create mode 100755 linux_configuration/scripts/single_use/utils/lib/analyze_repo_tools.sh diff --git a/docs/superpowers/contracts/analyze-repo-split-2026-08-16.json b/docs/superpowers/contracts/analyze-repo-split-2026-08-16.json new file mode 100644 index 00000000..bdc61460 --- /dev/null +++ b/docs/superpowers/contracts/analyze-repo-split-2026-08-16.json @@ -0,0 +1,19 @@ +{ + "title": "Split analyze_repo.sh (869 lines) into an entry script and five libs", + "objective": "Bring linux_configuration/scripts/single_use/utils/analyze_repo.sh under the repo-wide 250-line cap by extracting its helper functions and its per-STEP top-level blocks into lib/analyze_repo_*.sh, without changing what the script emits.", + "acceptance_criteria": [ + "analyze_repo.sh and every new lib/analyze_repo_*.sh file is at most 250 lines.", + "shellcheck reports zero findings across the entry script and all five libs, at the repo's no-severity-filter setting, with no suppression added.", + "The set of shell functions defined after sourcing is a superset of the original eight, adding only the six wrappers introduced around former top-level blocks: detect_languages, separate_code_and_comments, report_languages, analyze_symbols, analyze_cscope, analyze_clang_ast.", + "Every executable (non-comment, non-blank) line of the original file survives somewhere in the entry or a lib, except for lines deliberately edited to quote an assoc-array subscript, quote $TOP_N, or add a local declaration.", + "A stubbed run of the entry script's preamble defines all fourteen functions, proving each source line resolves at runtime rather than merely parsing.", + "The eight STEP headers the script prints are all still emitted, in the original order.", + "No lib sources another lib; the entry sources all five." + ], + "out_of_scope": [ + "Executing analyze_repo.sh, whose install_missing_tools calls sudo pacman, cargo install and makepkg; decision 6 forbids running it.", + "Defining KEYWORDS_SHELL and KEYWORDS_JAVA, which the report pass references but which have never existed. The lookup guards on a non-empty value so those two languages are silently skipped; defining them would change output and is a separate fix.", + "Fixing the unguarded cd calls or other pre-existing behaviour in the moved code beyond what shellcheck requires." + ], + "verifier": "shellcheck over the entry plus all five libs (expects zero findings), bash -n on each, bash ~/utils/scripts/check_file_length.sh over the six files (expects none listed), a line-set diff of the original against entry+libs showing no unintended loss, and a stubbed run of the entry preamble printing all fourteen function names." +} diff --git a/docs/superpowers/evidence/analyze-repo-split-2026-08-16.json b/docs/superpowers/evidence/analyze-repo-split-2026-08-16.json new file mode 100644 index 00000000..c44d7c2b --- /dev/null +++ b/docs/superpowers/evidence/analyze-repo-split-2026-08-16.json @@ -0,0 +1,64 @@ +{ + "intent": "Bring analyze_repo.sh under the 250-line cap by splitting it into a thin entry script plus five sourced libs, leaving the analysis output it produces unchanged.", + "scope": [ + "linux_configuration/scripts/single_use/utils/analyze_repo.sh (869 -> 231 lines)", + "New: lib/analyze_repo_{scan,tools,separate,report,symbols}.sh", + "Non-goal: running the script, which installs packages via sudo pacman / cargo / makepkg", + "Non-goal: defining the never-defined KEYWORDS_SHELL and KEYWORDS_JAVA tables" + ], + "changes": [ + "Extracted the output/discovery helpers (print_header, print_subheader, is_git_repo, find_files, count_files, fast_count) into lib/analyze_repo_scan.sh.", + "Extracted install_missing_tools, the 117-line package-manager dispatch, into lib/analyze_repo_tools.sh.", + "Wrapped the STEP 4 language-detection and comment/code-separation blocks as detect_languages and separate_code_and_comments in lib/analyze_repo_separate.sh; the HAS_* flags are set and read entirely within that file.", + "Wrapped the per-language keyword/function/import reporting as report_languages in lib/analyze_repo_report.sh, and moved the KEYWORDS_* tables in beside their only consumer.", + "Wrapped STEPs 5-7 as analyze_symbols, analyze_cscope and analyze_clang_ast in lib/analyze_repo_symbols.sh.", + "Passed the LANG_CODE_FILES map between the two STEP 4 passes explicitly by nameref and COMMENTS_TEMP by parameter, so no mutable state crosses a file boundary.", + "Fixed three unquoted $TOP_N expansions (SC2086) and quoted nine assoc-array subscripts that the nameref made shellcheck misread as arithmetic (SC2154). No suppression added." + ], + "verification": [ + { + "command": "shellcheck analyze_repo.sh lib/analyze_repo_*.sh", + "result": "pass", + "evidence": "Zero findings across all six files at the repo's no-severity-filter setting. An earlier boundary produced 23 findings (14 SC2034, 9 SC2154); those were cleared by moving the KEYWORDS_* tables and detect_languages beside their consumers, not by suppressing." + }, + { + "command": "bash -n on the entry and each lib", + "result": "pass", + "evidence": "All six parse. Run before linting, per the heredoc trap in docs/shell-split-recipes.md." + }, + { + "command": "bash ~/utils/scripts/check_file_length.sh --all", + "result": "pass", + "evidence": "entry 231, report 228, separate 186, tools 125, symbols 112, scan 86. None of the six appears in the violation list; repo total 69 -> 68." + }, + { + "command": "diff of the function set against a detached worktree at HEAD", + "result": "pass", + "evidence": "All eight original functions preserved (count_files, fast_count, find_files, install_missing_tools, is_git_repo, is_url, print_header, print_subheader). The six additions are exactly the wrappers named in the contract. Each function is defined exactly once across the six files." + }, + { + "command": "sorted line-set diff of HEAD:analyze_repo.sh against entry+libs concatenated", + "result": "pass", + "evidence": "After repair, no executable line is lost. The first pass caught a real regression this way: the `find \"$RESULTS_DIR/per_language/\" -maxdepth 1 -type f -printf` call at original line 719 fell into a one-line gap between two extraction ranges and was silently dropped. shellcheck and bash -n both passed on the broken version. Restored to the tail of report_languages." + }, + { + "command": "stubbed run of the entry preamble (source lines only, main flow replaced by declare -F)", + "result": "pass", + "evidence": "Prints all fourteen function names, proving each source line resolves at runtime. verify_shell_split.sh does not apply here: the script has no main \"$@\", so the by-hand fallback in docs/shell-split-recipes.md was used." + }, + { + "command": "grep -n 'source ' over the five libs", + "result": "pass", + "evidence": "No lib sources another; the entry sources all five. Only comment-text matches were returned." + } + ], + "risks": [ + "The script cannot be executed under decision 6, so verification is static plus a stubbed source. A behaviour change confined to the interior of a moved block would not be caught by the line-set diff, though that diff does prove no line was added, removed or reordered within a function body.", + "detect_languages now declares LANG_FILES as local rather than global. Confirmed safe: LANG_FILES was only ever read inside that same block. The HAS_* flags it sets remain global and are read by separate_code_and_comments in the same file.", + "The trap that cleans up COMMENTS_TEMP and /tmp/code_*.tmp moved into the entry script, where it is installed before the functions that create those files run. Same coverage as before, set one step earlier." + ], + "rollback": [ + "git revert the commit; analyze_repo.sh returns to its 869-line single-file form and the five libs disappear.", + "After rollback, re-run shellcheck on analyze_repo.sh to confirm it still lints clean, and confirm repo_to_study.sh still resolves ANALYZE_SCRIPT at scripts/single_use/utils/analyze_repo.sh, which this split did not move." + ] +} diff --git a/linux_configuration/scripts/single_use/utils/analyze_repo.sh b/linux_configuration/scripts/single_use/utils/analyze_repo.sh index 5e79b6f4..009a5aca 100755 --- a/linux_configuration/scripts/single_use/utils/analyze_repo.sh +++ b/linux_configuration/scripts/single_use/utils/analyze_repo.sh @@ -72,201 +72,26 @@ BLUE='\033[0;34m' YELLOW='\033[1;33m' NC='\033[0m' # No Color -print_header() { - echo "" - echo -e "${BLUE}════════════════════════════════════════════════════════════${NC}" - echo -e "${GREEN} $1${NC}" - echo -e "${BLUE}════════════════════════════════════════════════════════════${NC}" - echo "" -} -print_subheader() { - echo "" - echo -e "${YELLOW}--- $1 ---${NC}" - echo "" -} - -# Check if we're in a git repository -is_git_repo() { - git rev-parse --is-inside-work-tree &>/dev/null -} - -# Helper function to find files while respecting exclusions -# Usage: find_files "*.c" or find_files "*.py" "*.pyx" -find_files() { - local patterns=("$@") - - if [ "$RESPECT_GITIGNORE" = true ]; then - if is_git_repo; then - # Use git ls-files which respects .gitignore automatically - # This includes tracked files and untracked files not in .gitignore - local git_patterns=() - for pat in "${patterns[@]}"; do - git_patterns+=("$pat") - done - # Get tracked files + untracked (but not ignored) files - { - git ls-files -- "${git_patterns[@]}" 2>/dev/null - git ls-files --others --exclude-standard -- "${git_patterns[@]}" 2>/dev/null - } | sort -u - else - # Not a git repo - fall back to manual exclusion - local find_args=() - for i in "${!patterns[@]}"; do - if [ "$i" -eq 0 ]; then - find_args+=(-name "${patterns[$i]}") - else - find_args+=(-o -name "${patterns[$i]}") - fi - done - find . -type f \( "${find_args[@]}" \) 2>/dev/null | grep -Ev "/($EXCLUDE_DIRS)/" - fi - else - # No filtering - find all files - local find_args=() - for i in "${!patterns[@]}"; do - if [ "$i" -eq 0 ]; then - find_args+=(-name "${patterns[$i]}") - else - find_args+=(-o -name "${patterns[$i]}") - fi - done - find . -type f \( "${find_args[@]}" \) 2>/dev/null - fi -} - -# Count files matching pattern (respecting exclusions) -count_files() { - find_files "$@" | wc -l -} +#============================================================================== +# Libraries +#============================================================================== +# Sourced after the colour constants and configuration above, which they read. +source "$SCRIPT_DIR/lib/analyze_repo_scan.sh" +source "$SCRIPT_DIR/lib/analyze_repo_tools.sh" +source "$SCRIPT_DIR/lib/analyze_repo_separate.sh" +source "$SCRIPT_DIR/lib/analyze_repo_report.sh" +source "$SCRIPT_DIR/lib/analyze_repo_symbols.sh" #============================================================================== # STEP 0: Install Missing Tools #============================================================================== -install_missing_tools() { - local MISSING_TOOLS=() - local MISSING_AUR=() - - # Check for required tools - command -v git &>/dev/null || MISSING_TOOLS+=("git") - command -v ctags &>/dev/null || MISSING_TOOLS+=("ctags") - command -v cscope &>/dev/null || MISSING_TOOLS+=("cscope") - command -v clang &>/dev/null || MISSING_TOOLS+=("clang") - command -v ugrep &>/dev/null || MISSING_TOOLS+=("ugrep") - - # Check for AUR tools - command -v tokei &>/dev/null || MISSING_AUR+=("tokei") - command -v scc &>/dev/null || MISSING_AUR+=("scc") - - # Check for Rust 'counts' tool (install via cargo if missing) - if ! command -v counts &>/dev/null; then - if command -v cargo &>/dev/null; then - echo "Installing 'counts' via cargo (fast word counter)..." - cargo install counts 2>/dev/null || echo "Warning: counts install failed, will use Python fallback" - fi - fi - - # If nothing is missing, return - if [ ${#MISSING_TOOLS[@]} -eq 0 ] && [ ${#MISSING_AUR[@]} -eq 0 ]; then - echo -e "${GREEN}All required tools are installed.${NC}" - return 0 - fi - - echo -e "${YELLOW}Missing tools detected. Installing...${NC}" - - # Detect package manager - if command -v pacman &>/dev/null; then - # Arch Linux - if [ ${#MISSING_TOOLS[@]} -gt 0 ]; then - echo "Installing from official repos: ${MISSING_TOOLS[*]}" - sudo pacman -S --needed --noconfirm "${MISSING_TOOLS[@]}" - fi - - if [ ${#MISSING_AUR[@]} -gt 0 ]; then - # Find or install AUR helper - if command -v yay &>/dev/null; then - AUR_HELPER="yay" - elif command -v paru &>/dev/null; then - AUR_HELPER="paru" - else - echo "No AUR helper found. Installing yay..." - sudo pacman -S --needed --noconfirm base-devel git - TEMP_DIR=$(mktemp -d) - git clone https://aur.archlinux.org/yay.git "$TEMP_DIR/yay" - (cd "$TEMP_DIR/yay" && makepkg -si --noconfirm) - rm -rf "$TEMP_DIR" - AUR_HELPER="yay" - fi - - echo "Installing from AUR: ${MISSING_AUR[*]}" - $AUR_HELPER -S --needed --noconfirm "${MISSING_AUR[@]}" - fi - - elif command -v apt-get &>/dev/null; then - # Debian/Ubuntu - echo "Installing tools via apt..." - sudo apt-get update - - # Map tool names to package names - APT_PACKAGES=() - for tool in "${MISSING_TOOLS[@]}"; do - case $tool in - ctags) APT_PACKAGES+=("universal-ctags") ;; - ugrep) APT_PACKAGES+=("ugrep") ;; - *) APT_PACKAGES+=("$tool") ;; - esac - done - - [ ${#APT_PACKAGES[@]} -gt 0 ] && sudo apt-get install -y "${APT_PACKAGES[@]}" - - # Install tokei/scc via cargo or snap - for aur_tool in "${MISSING_AUR[@]}"; do - if command -v cargo &>/dev/null; then - echo "Installing $aur_tool via cargo..." - cargo install "$aur_tool" - elif command -v snap &>/dev/null; then - echo "Installing $aur_tool via snap..." - sudo snap install "$aur_tool" - else - echo -e "${YELLOW}Warning: Cannot install $aur_tool. Install cargo or snap first.${NC}" - fi - done - - elif command -v dnf &>/dev/null; then - # Fedora - echo "Installing tools via dnf..." - sudo dnf install -y "${MISSING_TOOLS[@]}" "${MISSING_AUR[@]}" 2>/dev/null || { - # tokei/scc might need cargo - for aur_tool in "${MISSING_AUR[@]}"; do - if command -v cargo &>/dev/null; then - cargo install "$aur_tool" - fi - done - } - - elif command -v brew &>/dev/null; then - # macOS with Homebrew - echo "Installing tools via brew..." - ALL_TOOLS=("${MISSING_TOOLS[@]}" "${MISSING_AUR[@]}") - brew install "${ALL_TOOLS[@]}" - - else - echo -e "${RED}Unknown package manager. Please install these tools manually:${NC}" - echo " Official: ${MISSING_TOOLS[*]}" - echo " Additional: ${MISSING_AUR[*]}" - exit 1 - fi - - echo -e "${GREEN}Tool installation complete.${NC}" -} - print_header "STEP 0: Checking/Installing Required Tools" install_missing_tools # Create directories mkdir -p "$WORK_DIR" "$RESULTS_DIR" -#============================================================================== # STEP 1: Clone or Use Local Repository #============================================================================== print_header "STEP 1: Repository Setup" @@ -331,499 +156,36 @@ scc --by-file --sort complexity . 2>/dev/null | head -20 | tee "$RESULTS_DIR/scc #============================================================================== print_header "STEP 4: Fast Keyword Analysis (Code vs Comments)" -# Helper function for fast word counting -# Uses 'counts' (Rust) if available, falls back to Python Counter -fast_count() { - local top_n="${1:-50}" - if command -v counts &>/dev/null; then - counts 2>/dev/null | head -n "$((top_n + 1))" | tail -n "$top_n" - else - python3 "$SCRIPT_DIR/fast_count.py" "$top_n" - fi -} +detect_languages -#------------------------------------------------------------------------------ -# Language Detection and Configuration -#------------------------------------------------------------------------------ -print_subheader "Detecting languages in repository..." - -if [ "$RESPECT_GITIGNORE" = true ]; then - if is_git_repo; then - echo -e "${YELLOW}Note: Respecting .gitignore (excludes node_modules, build outputs, etc.)${NC}" - else - echo -e "${YELLOW}Note: Excluding common directories (node_modules, .git, vendor, etc.)${NC}" - fi - echo " Use --no-ignore to include everything." - echo "" -fi - -# Count files by extension to detect primary languages (using helper) -declare -A LANG_FILES -LANG_FILES[c]=$(count_files "*.c") -LANG_FILES[cpp]=$(count_files "*.cpp" "*.cc" "*.cxx") -LANG_FILES[h]=$(count_files "*.h" "*.hpp" "*.hxx") -LANG_FILES[python]=$(count_files "*.py") -LANG_FILES[javascript]=$(count_files "*.js") -LANG_FILES[typescript]=$(count_files "*.ts" "*.tsx") -LANG_FILES[java]=$(count_files "*.java") -LANG_FILES[go]=$(count_files "*.go") -LANG_FILES[rust]=$(count_files "*.rs") -LANG_FILES[ruby]=$(count_files "*.rb") -LANG_FILES[shell]=$(count_files "*.sh" "*.bash") - -echo "Files found by language:" -for lang in c cpp h python javascript typescript java go rust ruby shell; do - count=${LANG_FILES[$lang]} - [ "$count" -gt 0 ] && echo " $lang: $count files" -done - -# Determine which language families are present -HAS_C_FAMILY=false -HAS_PYTHON=false -HAS_JS_FAMILY=false -HAS_SHELL=false -HAS_RUBY=false -HAS_GO=false -HAS_RUST=false -HAS_JAVA=false - -((LANG_FILES[c] + LANG_FILES[cpp] + LANG_FILES[h] > 0)) && HAS_C_FAMILY=true -((LANG_FILES[python] > 0)) && HAS_PYTHON=true -((LANG_FILES[javascript] + LANG_FILES[typescript] > 0)) && HAS_JS_FAMILY=true -((LANG_FILES[shell] > 0)) && HAS_SHELL=true -((LANG_FILES[ruby] > 0)) && HAS_RUBY=true -((LANG_FILES[go] > 0)) && HAS_GO=true -((LANG_FILES[rust] > 0)) && HAS_RUST=true -((LANG_FILES[java] > 0)) && HAS_JAVA=true - -#------------------------------------------------------------------------------ -# Language-specific keyword definitions -#------------------------------------------------------------------------------ -# C/C++ keywords -KEYWORDS_C="auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|inline|restrict|_Bool|_Complex|_Imaginary" -KEYWORDS_CPP="$KEYWORDS_C|alignas|alignof|and|and_eq|asm|atomic_cancel|atomic_commit|atomic_noexcept|bitand|bitor|bool|catch|char16_t|char32_t|char8_t|class|co_await|co_return|co_yield|compl|concept|const_cast|consteval|constexpr|constinit|decltype|delete|dynamic_cast|explicit|export|false|friend|mutable|namespace|new|noexcept|not|not_eq|nullptr|operator|or|or_eq|override|private|protected|public|reflexpr|reinterpret_cast|requires|static_assert|static_cast|synchronized|template|this|thread_local|throw|true|try|typeid|typename|using|virtual|wchar_t|xor|xor_eq" - -# Python keywords -KEYWORDS_PYTHON="False|None|True|and|as|assert|async|await|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield" - -# JavaScript/TypeScript keywords -KEYWORDS_JS="abstract|arguments|await|boolean|break|byte|case|catch|char|class|const|continue|debugger|default|delete|do|double|else|enum|eval|export|extends|false|final|finally|float|for|function|goto|if|implements|import|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|super|switch|synchronized|this|throw|throws|transient|true|try|typeof|undefined|var|void|volatile|while|with|yield" -KEYWORDS_TS="$KEYWORDS_JS|any|as|asserts|bigint|declare|get|infer|intrinsic|is|keyof|module|namespace|never|out|override|readonly|require|set|string|symbol|type|unique|unknown" - -# Go keywords -KEYWORDS_GO="break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go|goto|if|import|interface|map|package|range|return|select|struct|switch|type|var" - -# Rust keywords -KEYWORDS_RUST="as|async|await|break|const|continue|crate|dyn|else|enum|extern|false|fn|for|if|impl|in|let|loop|match|mod|move|mut|pub|ref|return|self|Self|static|struct|super|trait|true|type|unsafe|use|where|while" - -# Ruby keywords -KEYWORDS_RUBY="BEGIN|END|alias|and|begin|break|case|class|def|defined|do|else|elsif|end|ensure|false|for|if|in|module|next|nil|not|or|redo|rescue|retry|return|self|super|then|true|undef|unless|until|when|while|yield" #------------------------------------------------------------------------------ # Multi-language comment processing - KEEP LANGUAGES SEPARATE #------------------------------------------------------------------------------ print_subheader "Processing source files (separating code from comments)..." -# Create per-language output directory -mkdir -p "$RESULTS_DIR/per_language" COMMENTS_TEMP=$(mktemp) trap 'rm -f "$COMMENTS_TEMP" /tmp/code_*.tmp 2>/dev/null' EXIT - declare -A LANG_CODE_FILES -# Process C/C++ files -if $HAS_C_FAMILY; then - echo "Processing C/C++ files..." - LANG_CODE_FILES[c_cpp]=$(mktemp /tmp/code_c_cpp.XXXXXX.tmp) - find_files "*.c" "*.cpp" "*.cc" "*.cxx" "*.h" "*.hpp" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[c_cpp]}" +separate_code_and_comments LANG_CODE_FILES "$COMMENTS_TEMP" +report_languages LANG_CODE_FILES "$COMMENTS_TEMP" - # Extract and strip C-style comments - perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[c_cpp]}" >>"$COMMENTS_TEMP" - perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[c_cpp]}" >"${LANG_CODE_FILES[c_cpp]}.clean" - mv "${LANG_CODE_FILES[c_cpp]}.clean" "${LANG_CODE_FILES[c_cpp]}" -fi - -# Process JavaScript files (separate from TypeScript) -if $HAS_JS_FAMILY; then - echo "Processing JavaScript files..." - LANG_CODE_FILES[javascript]=$(mktemp /tmp/code_js.XXXXXX.tmp) - find_files "*.js" "*.jsx" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[javascript]}" - - echo "Processing TypeScript files..." - LANG_CODE_FILES[typescript]=$(mktemp /tmp/code_ts.XXXXXX.tmp) - find_files "*.ts" "*.tsx" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[typescript]}" - - # Extract and strip comments from both - for lang_file in "${LANG_CODE_FILES[javascript]}" "${LANG_CODE_FILES[typescript]}"; do - [ ! -s "$lang_file" ] && continue - perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "$lang_file" >>"$COMMENTS_TEMP" - perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "$lang_file" >"${lang_file}.clean" - mv "${lang_file}.clean" "$lang_file" - done -fi - -# Process Python files -if $HAS_PYTHON; then - echo "Processing Python files..." - LANG_CODE_FILES[python]=$(mktemp /tmp/code_python.XXXXXX.tmp) - find_files "*.py" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[python]}" - - perl -ne 'if (/^\s*#(.*)/) { print "$1\n"; } elsif (/#(.*)$/) { print "$1\n"; }' "${LANG_CODE_FILES[python]}" >>"$COMMENTS_TEMP" - perl -0777 -ne 'while (/"""(.+?)"""/gs) { print "$1\n"; } while (/'"'"''"'"''"'"'(.+?)'"'"''"'"''"'"'/gs) { print "$1\n"; }' "${LANG_CODE_FILES[python]}" >>"$COMMENTS_TEMP" - perl -pe 's/#.*$//' "${LANG_CODE_FILES[python]}" | perl -0777 -pe 's/""".*?"""//gs; s/'"'"''"'"''"'"'.*?'"'"''"'"''"'"'//gs' >"${LANG_CODE_FILES[python]}.clean" - mv "${LANG_CODE_FILES[python]}.clean" "${LANG_CODE_FILES[python]}" -fi - -# Process Go files -if $HAS_GO; then - echo "Processing Go files..." - LANG_CODE_FILES[go]=$(mktemp /tmp/code_go.XXXXXX.tmp) - find_files "*.go" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[go]}" - - perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[go]}" >>"$COMMENTS_TEMP" - perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[go]}" >"${LANG_CODE_FILES[go]}.clean" - mv "${LANG_CODE_FILES[go]}.clean" "${LANG_CODE_FILES[go]}" -fi - -# Process Rust files -if $HAS_RUST; then - echo "Processing Rust files..." - LANG_CODE_FILES[rust]=$(mktemp /tmp/code_rust.XXXXXX.tmp) - find_files "*.rs" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[rust]}" - - perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[rust]}" >>"$COMMENTS_TEMP" - perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[rust]}" >"${LANG_CODE_FILES[rust]}.clean" - mv "${LANG_CODE_FILES[rust]}.clean" "${LANG_CODE_FILES[rust]}" -fi - -# Process Ruby files -if $HAS_RUBY; then - echo "Processing Ruby files..." - LANG_CODE_FILES[ruby]=$(mktemp /tmp/code_ruby.XXXXXX.tmp) - find_files "*.rb" | head -5000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[ruby]}" - - perl -ne 'if (/#(.*)$/) { print "$1\n"; }' "${LANG_CODE_FILES[ruby]}" >>"$COMMENTS_TEMP" - perl -0777 -ne 'while (/=begin(.+?)=end/gs) { print "$1\n"; }' "${LANG_CODE_FILES[ruby]}" >>"$COMMENTS_TEMP" - perl -pe 's/#.*$//' "${LANG_CODE_FILES[ruby]}" | perl -0777 -pe 's/=begin.*?=end//gs' >"${LANG_CODE_FILES[ruby]}.clean" - mv "${LANG_CODE_FILES[ruby]}.clean" "${LANG_CODE_FILES[ruby]}" -fi - -# Process Shell files -if $HAS_SHELL; then - echo "Processing Shell files..." - LANG_CODE_FILES[shell]=$(mktemp /tmp/code_shell.XXXXXX.tmp) - find_files "*.sh" "*.bash" | head -5000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[shell]}" - - perl -ne 'if (/^\s*#(.*)/ && !/^#!/) { print "$1\n"; } elsif (/#(.*)$/) { print "$1\n"; }' "${LANG_CODE_FILES[shell]}" >>"$COMMENTS_TEMP" - perl -pe 's/#.*$//' "${LANG_CODE_FILES[shell]}" >"${LANG_CODE_FILES[shell]}.clean" - mv "${LANG_CODE_FILES[shell]}.clean" "${LANG_CODE_FILES[shell]}" -fi - -# Process Java files -if $HAS_JAVA; then - echo "Processing Java files..." - LANG_CODE_FILES[java]=$(mktemp /tmp/code_java.XXXXXX.tmp) - find_files "*.java" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[java]}" - - perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[java]}" >>"$COMMENTS_TEMP" - perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[java]}" >"${LANG_CODE_FILES[java]}.clean" - mv "${LANG_CODE_FILES[java]}.clean" "${LANG_CODE_FILES[java]}" -fi - -COMMENT_LINES=$(wc -l <"$COMMENTS_TEMP") -echo "" -echo "Processed languages: ${!LANG_CODE_FILES[*]}" -echo "Total comment lines: $COMMENT_LINES" - -#------------------------------------------------------------------------------ -# Per-Language Keyword Analysis - Each language gets its own file -#------------------------------------------------------------------------------ -print_subheader "Per-Language Keyword Analysis" - -# Map language names to keyword variables -declare -A LANG_KEYWORDS -LANG_KEYWORDS[c_cpp]="$KEYWORDS_CPP" -LANG_KEYWORDS[python]="$KEYWORDS_PYTHON" -LANG_KEYWORDS[javascript]="$KEYWORDS_JS" -LANG_KEYWORDS[typescript]="$KEYWORDS_TS" -LANG_KEYWORDS[go]="$KEYWORDS_GO" -LANG_KEYWORDS[rust]="$KEYWORDS_RUST" -LANG_KEYWORDS[ruby]="$KEYWORDS_RUBY" -LANG_KEYWORDS[shell]="$KEYWORDS_SHELL" -LANG_KEYWORDS[java]="$KEYWORDS_JAVA" - -# Analyze each language separately -for lang in "${!LANG_CODE_FILES[@]}"; do - code_file="${LANG_CODE_FILES[$lang]}" - keywords="${LANG_KEYWORDS[$lang]}" - output_file="$RESULTS_DIR/per_language/keywords_${lang}.txt" - - if [ -f "$code_file" ] && [ -s "$code_file" ] && [ -n "$keywords" ]; then - echo "" - echo -e "${YELLOW}=== $lang Keywords ===${NC}" - ugrep -o "\b($keywords)\b" "$code_file" 2>/dev/null | - fast_count 50 | - tee "$output_file" - fi -done - -#------------------------------------------------------------------------------ -# Per-Language Function Analysis -#------------------------------------------------------------------------------ -print_subheader "Per-Language Function Calls" - -for lang in "${!LANG_CODE_FILES[@]}"; do - code_file="${LANG_CODE_FILES[$lang]}" - output_file="$RESULTS_DIR/per_language/functions_${lang}.txt" - - if [ -f "$code_file" ] && [ -s "$code_file" ]; then - echo "" - echo -e "${YELLOW}=== $lang Functions ===${NC}" - ugrep -o '\b[a-zA-Z_][a-zA-Z0-9_]*\s*\(' "$code_file" 2>/dev/null | - sed 's/\s*(//' | - grep -vE '^(if|for|while|switch|catch|elif)$' | - fast_count 30 | - tee "$output_file" - fi -done - -#------------------------------------------------------------------------------ -# Per-Language Import Analysis -#------------------------------------------------------------------------------ -print_subheader "Per-Language Imports/Includes" - -# C/C++ includes -if [ -n "${LANG_CODE_FILES[c_cpp]}" ] && [ -s "${LANG_CODE_FILES[c_cpp]}" ]; then - echo -e "${YELLOW}=== C/C++ Includes ===${NC}" - ugrep -o '#include\s*[<"][^>"]+[>"]' "${LANG_CODE_FILES[c_cpp]}" 2>/dev/null | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_c_cpp.txt" -fi - -# Python imports -if [ -n "${LANG_CODE_FILES[python]}" ] && [ -s "${LANG_CODE_FILES[python]}" ]; then - echo "" - echo -e "${YELLOW}=== Python Imports ===${NC}" - ugrep -o '^\s*(from\s+\S+\s+import\s+\S+|import\s+\S+)' "${LANG_CODE_FILES[python]}" 2>/dev/null | - sed 's/^\s*//' | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_python.txt" -fi - -# JavaScript imports -if [ -n "${LANG_CODE_FILES[javascript]}" ] && [ -s "${LANG_CODE_FILES[javascript]}" ]; then - echo "" - echo -e "${YELLOW}=== JavaScript Imports ===${NC}" - ugrep -o "(import\s+.*\s+from\s+['\"][^'\"]+['\"]|require\s*\(['\"][^'\"]+['\"]\))" "${LANG_CODE_FILES[javascript]}" 2>/dev/null | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_javascript.txt" -fi - -# TypeScript imports -if [ -n "${LANG_CODE_FILES[typescript]}" ] && [ -s "${LANG_CODE_FILES[typescript]}" ]; then - echo "" - echo -e "${YELLOW}=== TypeScript Imports ===${NC}" - ugrep -o "(import\s+.*\s+from\s+['\"][^'\"]+['\"]|require\s*\(['\"][^'\"]+['\"]\))" "${LANG_CODE_FILES[typescript]}" 2>/dev/null | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_typescript.txt" -fi - -# Go imports -if [ -n "${LANG_CODE_FILES[go]}" ] && [ -s "${LANG_CODE_FILES[go]}" ]; then - echo "" - echo -e "${YELLOW}=== Go Imports ===${NC}" - ugrep -o '"[^"]+/[^"]+"' "${LANG_CODE_FILES[go]}" 2>/dev/null | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_go.txt" -fi - -# Rust use statements -if [ -n "${LANG_CODE_FILES[rust]}" ] && [ -s "${LANG_CODE_FILES[rust]}" ]; then - echo "" - echo -e "${YELLOW}=== Rust Use Statements ===${NC}" - ugrep -o '^\s*use\s+[^;]+' "${LANG_CODE_FILES[rust]}" 2>/dev/null | - sed 's/^\s*//' | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_rust.txt" -fi - -# Java imports -if [ -n "${LANG_CODE_FILES[java]}" ] && [ -s "${LANG_CODE_FILES[java]}" ]; then - echo "" - echo -e "${YELLOW}=== Java Imports ===${NC}" - ugrep -o '^\s*import\s+[^;]+' "${LANG_CODE_FILES[java]}" 2>/dev/null | - sed 's/^\s*//' | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_java.txt" -fi - -# Ruby requires -if [ -n "${LANG_CODE_FILES[ruby]}" ] && [ -s "${LANG_CODE_FILES[ruby]}" ]; then - echo "" - echo -e "${YELLOW}=== Ruby Requires ===${NC}" - ugrep -o "(require\s+['\"][^'\"]+['\"]|require_relative\s+['\"][^'\"]+['\"])" "${LANG_CODE_FILES[ruby]}" 2>/dev/null | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_ruby.txt" -fi - -# Shell sources -if [ -n "${LANG_CODE_FILES[shell]}" ] && [ -s "${LANG_CODE_FILES[shell]}" ]; then - echo "" - echo -e "${YELLOW}=== Shell Sources ===${NC}" - ugrep -o '(source\s+[^\s]+|\.\s+[^\s]+)' "${LANG_CODE_FILES[shell]}" 2>/dev/null | - fast_count 30 | - tee "$RESULTS_DIR/per_language/imports_shell.txt" -fi - -#------------------------------------------------------------------------------ -# Combined Analysis (for overview/backward compatibility) -#------------------------------------------------------------------------------ -print_subheader "Combined Code Identifiers (all languages)" - -# Create combined CODE_TEMP -CODE_TEMP=$(mktemp) -for lang_file in "${LANG_CODE_FILES[@]}"; do - [ -f "$lang_file" ] && cat "$lang_file" >>"$CODE_TEMP" -done - -ugrep -o '\b[a-zA-Z_][a-zA-Z0-9_]*\b' "$CODE_TEMP" 2>/dev/null | - fast_count $TOP_N | - tee "$RESULTS_DIR/code_identifiers.txt" - -print_subheader "Most Used Words in COMMENTS" -ugrep -o '\b[a-zA-Z_][a-zA-Z0-9_]*\b' "$COMMENTS_TEMP" 2>/dev/null | - fast_count $TOP_N | - tee "$RESULTS_DIR/comment_words.txt" - -# Create combined files from per-language analysis (for backward compatibility) -{ - echo "# Combined keywords from all languages" - echo "# Format: count keyword (from per_language/keywords_*.txt)" - cat "$RESULTS_DIR/per_language"/keywords_*.txt 2>/dev/null | grep -v '^$' | sort -t' ' -k1 -nr | head -100 -} >"$RESULTS_DIR/grep_keywords.txt" - -{ - echo "# Combined functions from all languages" - echo "# See per_language/functions_*.txt for language-specific breakdown" - cat "$RESULTS_DIR/per_language"/functions_*.txt 2>/dev/null | grep -v '^$' | sort -t' ' -k1 -nr | head -100 -} >"$RESULTS_DIR/grep_function_calls.txt" - -{ - echo "# Combined imports from all languages" - echo "# See per_language/imports_*.txt for language-specific breakdown" - cat "$RESULTS_DIR/per_language"/imports_*.txt 2>/dev/null | grep -v '^$' | sort -t' ' -k1 -nr | head -100 -} >"$RESULTS_DIR/grep_imports.txt" - -# List what per-language files were created -echo "" -echo "Per-language analysis files created:" -find "$RESULTS_DIR/per_language/" -maxdepth 1 -type f -printf ' %f\n' 2>/dev/null || true - -print_subheader "Generating tags (this may take a while)..." - -# Generate tags for different kinds -ctags -R --languages=C,C++ --c-kinds=+fp --fields=+lK -f "$RESULTS_DIR/tags" . 2>/dev/null || true - -if [ -f "$RESULTS_DIR/tags" ]; then - TOTAL_TAGS=$(grep -ac '^[^!]' "$RESULTS_DIR/tags" 2>/dev/null || echo "0") - echo "Total symbols found: $TOTAL_TAGS" - - print_subheader "Most Common Symbol Names" - # Fast: use cut + counts instead of awk + sort | uniq - # -a flag treats tags file as text (may contain binary-like patterns) - grep -a '^[^!]' "$RESULTS_DIR/tags" | cut -f1 | fast_count $TOP_N | - tee "$RESULTS_DIR/ctags_symbols.txt" - - print_subheader "Symbol Types Distribution" - # Fast: extract single-letter kind code after ;" and count - grep -aoP ';"\t\K[a-z]' "$RESULTS_DIR/tags" 2>/dev/null | fast_count 20 | while read -r count kind; do - case $kind in - f) echo "$count functions" ;; - v) echo "$count variables" ;; - s) echo "$count structs" ;; - t) echo "$count typedefs" ;; - e) echo "$count enum values" ;; - g) echo "$count enums" ;; - m) echo "$count struct/union members" ;; - d) echo "$count macro definitions" ;; - p) echo "$count function prototypes" ;; - u) echo "$count unions" ;; - c) echo "$count classes" ;; - n) echo "$count namespaces" ;; - *) echo "$count kind=$kind" ;; - esac - done | tee "$RESULTS_DIR/ctags_kinds.txt" -fi +#============================================================================== +# STEP 5: ctags Symbol Analysis +#============================================================================== +analyze_symbols #============================================================================== # STEP 6: cscope Analysis #============================================================================== print_header "STEP 6: cscope Database Analysis" - -print_subheader "Building cscope database..." - -# Find all C source files (respecting .gitignore if available) -if [ "$RESPECT_GITIGNORE" = true ] && is_git_repo; then - { - git ls-files -- '*.c' '*.h' 2>/dev/null - git ls-files --others --exclude-standard -- '*.c' '*.h' 2>/dev/null - } | sort -u >"$RESULTS_DIR/cscope.files" -elif [ "$RESPECT_GITIGNORE" = true ]; then - find . \( -name "*.c" -o -name "*.h" \) -type f 2>/dev/null | grep -Ev "/($EXCLUDE_DIRS)/" >"$RESULTS_DIR/cscope.files" -else - find . \( -name "*.c" -o -name "*.h" \) -type f >"$RESULTS_DIR/cscope.files" 2>/dev/null -fi -FILE_COUNT=$(wc -l <"$RESULTS_DIR/cscope.files") -echo "Found $FILE_COUNT source files" - -# Build cscope database (can take a while for large repos) -echo "Building database (this may take several minutes for Linux kernel)..." -cscope -b -q -i "$RESULTS_DIR/cscope.files" -f "$RESULTS_DIR/cscope.out" 2>/dev/null || true - -if [ -f "$RESULTS_DIR/cscope.out" ]; then - echo "Database built successfully" - echo "Database size: $(du -sh "$RESULTS_DIR/cscope.out" | cut -f1)" - - print_subheader "Example: Finding callers of 'printk' function" - cscope -d -f "$RESULTS_DIR/cscope.out" -L -3 printk 2>/dev/null | head -20 || echo "No results" - - print_subheader "Example: Finding definition of 'struct file'" - cscope -d -f "$RESULTS_DIR/cscope.out" -L -1 "struct file" 2>/dev/null | head -10 || echo "No results" -fi +analyze_cscope #============================================================================== # STEP 7: clang AST Analysis (if available) #============================================================================== print_header "STEP 7: clang-based Analysis (AST-level)" - -print_subheader "Analyzing a sample file with clang AST dump" - -# Find a simple C file to analyze (respecting .gitignore) -if [ "$RESPECT_GITIGNORE" = true ] && is_git_repo; then - SAMPLE_FILE=$(git ls-files -- '*.c' 2>/dev/null | head -20 | while read -r f; do - [ -f "$f" ] && [ "$(stat -c%s "$f" 2>/dev/null || echo 999999)" -lt 51200 ] && echo "$f" - done | head -1) -elif [ "$RESPECT_GITIGNORE" = true ]; then - SAMPLE_FILE=$(find . -name "*.c" -size -50k -type f 2>/dev/null | grep -Ev "/($EXCLUDE_DIRS)/" | head -1) -else - SAMPLE_FILE=$(find . -name "*.c" -size -50k 2>/dev/null | head -1) -fi - -if [ -n "$SAMPLE_FILE" ]; then - echo "Sample file: $SAMPLE_FILE" - echo "" - echo "Function declarations in this file:" - clang -Xclang -ast-dump -fsyntax-only "$SAMPLE_FILE" 2>/dev/null | - grep -E "FunctionDecl.*<.*>" | - head -20 | - sed 's/.*FunctionDecl.*<[^>]*> / /' | - tee "$RESULTS_DIR/clang_sample_functions.txt" || echo "Analysis failed (missing headers)" -fi - -print_subheader "Note: Full clang analysis requires compile_commands.json" -echo "For proper AST analysis of the Linux kernel, you need to:" -echo " 1. Configure the kernel: make defconfig" -echo " 2. Generate compile_commands.json: make compile_commands.json" -echo " 3. Use clang-query or clang-check with the database" +analyze_clang_ast #============================================================================== # STEP 8: Summary diff --git a/linux_configuration/scripts/single_use/utils/lib/analyze_repo_report.sh b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_report.sh new file mode 100755 index 00000000..9fec4ae7 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_report.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# Per-language keyword, function-call and import reporting, plus the combined +# cross-language rollups. +# +# Sourced by analyze_repo.sh; split out to keep it under the 250-line cap. +# Sourced rather than run, so it inherits the caller's strict mode and reads +# RESULTS_DIR, TOP_N and the colour constants. +# +# The KEYWORDS_* tables live here rather than in their own data file because +# report_languages below is their only consumer; keeping definition and use in +# one file is what lets shellcheck see them as used. +# +# Note: LANG_KEYWORDS also indexes shell and java, for which no table has ever +# been defined. The lookup guards on a non-empty value, so those two languages +# are skipped rather than failing. Pre-existing; preserved by this split. + +#------------------------------------------------------------------------------ +# C/C++ keywords +KEYWORDS_C="auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|inline|restrict|_Bool|_Complex|_Imaginary" +KEYWORDS_CPP="$KEYWORDS_C|alignas|alignof|and|and_eq|asm|atomic_cancel|atomic_commit|atomic_noexcept|bitand|bitor|bool|catch|char16_t|char32_t|char8_t|class|co_await|co_return|co_yield|compl|concept|const_cast|consteval|constexpr|constinit|decltype|delete|dynamic_cast|explicit|export|false|friend|mutable|namespace|new|noexcept|not|not_eq|nullptr|operator|or|or_eq|override|private|protected|public|reflexpr|reinterpret_cast|requires|static_assert|static_cast|synchronized|template|this|thread_local|throw|true|try|typeid|typename|using|virtual|wchar_t|xor|xor_eq" + +# Python keywords +KEYWORDS_PYTHON="False|None|True|and|as|assert|async|await|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield" + +# JavaScript/TypeScript keywords +KEYWORDS_JS="abstract|arguments|await|boolean|break|byte|case|catch|char|class|const|continue|debugger|default|delete|do|double|else|enum|eval|export|extends|false|final|finally|float|for|function|goto|if|implements|import|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|super|switch|synchronized|this|throw|throws|transient|true|try|typeof|undefined|var|void|volatile|while|with|yield" +KEYWORDS_TS="$KEYWORDS_JS|any|as|asserts|bigint|declare|get|infer|intrinsic|is|keyof|module|namespace|never|out|override|readonly|require|set|string|symbol|type|unique|unknown" + +# Go keywords +KEYWORDS_GO="break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go|goto|if|import|interface|map|package|range|return|select|struct|switch|type|var" + +# Rust keywords +KEYWORDS_RUST="as|async|await|break|const|continue|crate|dyn|else|enum|extern|false|fn|for|if|impl|in|let|loop|match|mod|move|mut|pub|ref|return|self|Self|static|struct|super|trait|true|type|unsafe|use|where|while" + +# Ruby keywords +KEYWORDS_RUBY="BEGIN|END|alias|and|begin|break|case|class|def|defined|do|else|elsif|end|ensure|false|for|if|in|module|next|nil|not|or|redo|rescue|retry|return|self|super|then|true|undef|unless|until|when|while|yield" +#------------------------------------------------------------------------------ +# Multi-language comment processing - KEEP LANGUAGES SEPARATE + +# report_languages +# Takes the map built by separate_code_and_comments explicitly, so no mutable +# state crosses the file boundary between the two passes. +report_languages() { + local -n LANG_CODE_FILES="$1" + local COMMENTS_TEMP="$2" + local lang code_file keywords output_file lang_file CODE_TEMP + + print_subheader "Per-Language Keyword Analysis" + + # Map language names to keyword variables + declare -A LANG_KEYWORDS + LANG_KEYWORDS[c_cpp]="$KEYWORDS_CPP" + LANG_KEYWORDS[python]="$KEYWORDS_PYTHON" + LANG_KEYWORDS[javascript]="$KEYWORDS_JS" + LANG_KEYWORDS[typescript]="$KEYWORDS_TS" + LANG_KEYWORDS[go]="$KEYWORDS_GO" + LANG_KEYWORDS[rust]="$KEYWORDS_RUST" + LANG_KEYWORDS[ruby]="$KEYWORDS_RUBY" + LANG_KEYWORDS[shell]="$KEYWORDS_SHELL" + LANG_KEYWORDS[java]="$KEYWORDS_JAVA" + + # Analyze each language separately + for lang in "${!LANG_CODE_FILES[@]}"; do + code_file="${LANG_CODE_FILES[$lang]}" + keywords="${LANG_KEYWORDS[$lang]}" + output_file="$RESULTS_DIR/per_language/keywords_${lang}.txt" + + if [ -f "$code_file" ] && [ -s "$code_file" ] && [ -n "$keywords" ]; then + echo "" + echo -e "${YELLOW}=== $lang Keywords ===${NC}" + ugrep -o "\b($keywords)\b" "$code_file" 2>/dev/null | + fast_count 50 | + tee "$output_file" + fi + done + + #------------------------------------------------------------------------------ + # Per-Language Function Analysis + #------------------------------------------------------------------------------ + print_subheader "Per-Language Function Calls" + + for lang in "${!LANG_CODE_FILES[@]}"; do + code_file="${LANG_CODE_FILES[$lang]}" + output_file="$RESULTS_DIR/per_language/functions_${lang}.txt" + + if [ -f "$code_file" ] && [ -s "$code_file" ]; then + echo "" + echo -e "${YELLOW}=== $lang Functions ===${NC}" + ugrep -o '\b[a-zA-Z_][a-zA-Z0-9_]*\s*\(' "$code_file" 2>/dev/null | + sed 's/\s*(//' | + grep -vE '^(if|for|while|switch|catch|elif)$' | + fast_count 30 | + tee "$output_file" + fi + done + + #------------------------------------------------------------------------------ + # Per-Language Import Analysis + #------------------------------------------------------------------------------ + print_subheader "Per-Language Imports/Includes" + + # C/C++ includes + if [ -n "${LANG_CODE_FILES[c_cpp]}" ] && [ -s "${LANG_CODE_FILES[c_cpp]}" ]; then + echo -e "${YELLOW}=== C/C++ Includes ===${NC}" + ugrep -o '#include\s*[<"][^>"]+[>"]' "${LANG_CODE_FILES[c_cpp]}" 2>/dev/null | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_c_cpp.txt" + fi + + # Python imports + if [ -n "${LANG_CODE_FILES[python]}" ] && [ -s "${LANG_CODE_FILES[python]}" ]; then + echo "" + echo -e "${YELLOW}=== Python Imports ===${NC}" + ugrep -o '^\s*(from\s+\S+\s+import\s+\S+|import\s+\S+)' "${LANG_CODE_FILES[python]}" 2>/dev/null | + sed 's/^\s*//' | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_python.txt" + fi + + # JavaScript imports + if [ -n "${LANG_CODE_FILES[javascript]}" ] && [ -s "${LANG_CODE_FILES[javascript]}" ]; then + echo "" + echo -e "${YELLOW}=== JavaScript Imports ===${NC}" + ugrep -o "(import\s+.*\s+from\s+['\"][^'\"]+['\"]|require\s*\(['\"][^'\"]+['\"]\))" "${LANG_CODE_FILES[javascript]}" 2>/dev/null | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_javascript.txt" + fi + + # TypeScript imports + if [ -n "${LANG_CODE_FILES[typescript]}" ] && [ -s "${LANG_CODE_FILES[typescript]}" ]; then + echo "" + echo -e "${YELLOW}=== TypeScript Imports ===${NC}" + ugrep -o "(import\s+.*\s+from\s+['\"][^'\"]+['\"]|require\s*\(['\"][^'\"]+['\"]\))" "${LANG_CODE_FILES[typescript]}" 2>/dev/null | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_typescript.txt" + fi + + # Go imports + if [ -n "${LANG_CODE_FILES[go]}" ] && [ -s "${LANG_CODE_FILES[go]}" ]; then + echo "" + echo -e "${YELLOW}=== Go Imports ===${NC}" + ugrep -o '"[^"]+/[^"]+"' "${LANG_CODE_FILES[go]}" 2>/dev/null | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_go.txt" + fi + + # Rust use statements + if [ -n "${LANG_CODE_FILES[rust]}" ] && [ -s "${LANG_CODE_FILES[rust]}" ]; then + echo "" + echo -e "${YELLOW}=== Rust Use Statements ===${NC}" + ugrep -o '^\s*use\s+[^;]+' "${LANG_CODE_FILES[rust]}" 2>/dev/null | + sed 's/^\s*//' | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_rust.txt" + fi + + # Java imports + if [ -n "${LANG_CODE_FILES[java]}" ] && [ -s "${LANG_CODE_FILES[java]}" ]; then + echo "" + echo -e "${YELLOW}=== Java Imports ===${NC}" + ugrep -o '^\s*import\s+[^;]+' "${LANG_CODE_FILES[java]}" 2>/dev/null | + sed 's/^\s*//' | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_java.txt" + fi + + # Ruby requires + if [ -n "${LANG_CODE_FILES[ruby]}" ] && [ -s "${LANG_CODE_FILES[ruby]}" ]; then + echo "" + echo -e "${YELLOW}=== Ruby Requires ===${NC}" + ugrep -o "(require\s+['\"][^'\"]+['\"]|require_relative\s+['\"][^'\"]+['\"])" "${LANG_CODE_FILES[ruby]}" 2>/dev/null | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_ruby.txt" + fi + + # Shell sources + if [ -n "${LANG_CODE_FILES[shell]}" ] && [ -s "${LANG_CODE_FILES[shell]}" ]; then + echo "" + echo -e "${YELLOW}=== Shell Sources ===${NC}" + ugrep -o '(source\s+[^\s]+|\.\s+[^\s]+)' "${LANG_CODE_FILES[shell]}" 2>/dev/null | + fast_count 30 | + tee "$RESULTS_DIR/per_language/imports_shell.txt" + fi + + #------------------------------------------------------------------------------ + # Combined Analysis (for overview/backward compatibility) + #------------------------------------------------------------------------------ + print_subheader "Combined Code Identifiers (all languages)" + + # Create combined CODE_TEMP + CODE_TEMP=$(mktemp) + for lang_file in "${LANG_CODE_FILES[@]}"; do + [ -f "$lang_file" ] && cat "$lang_file" >>"$CODE_TEMP" + done + + ugrep -o '\b[a-zA-Z_][a-zA-Z0-9_]*\b' "$CODE_TEMP" 2>/dev/null | + fast_count "$TOP_N" | + tee "$RESULTS_DIR/code_identifiers.txt" + + print_subheader "Most Used Words in COMMENTS" + ugrep -o '\b[a-zA-Z_][a-zA-Z0-9_]*\b' "$COMMENTS_TEMP" 2>/dev/null | + fast_count "$TOP_N" | + tee "$RESULTS_DIR/comment_words.txt" + + # Create combined files from per-language analysis (for backward compatibility) + { + echo "# Combined keywords from all languages" + echo "# Format: count keyword (from per_language/keywords_*.txt)" + cat "$RESULTS_DIR/per_language"/keywords_*.txt 2>/dev/null | grep -v '^$' | sort -t' ' -k1 -nr | head -100 + } >"$RESULTS_DIR/grep_keywords.txt" + + { + echo "# Combined functions from all languages" + echo "# See per_language/functions_*.txt for language-specific breakdown" + cat "$RESULTS_DIR/per_language"/functions_*.txt 2>/dev/null | grep -v '^$' | sort -t' ' -k1 -nr | head -100 + } >"$RESULTS_DIR/grep_function_calls.txt" + + { + echo "# Combined imports from all languages" + echo "# See per_language/imports_*.txt for language-specific breakdown" + cat "$RESULTS_DIR/per_language"/imports_*.txt 2>/dev/null | grep -v '^$' | sort -t' ' -k1 -nr | head -100 + } >"$RESULTS_DIR/grep_imports.txt" + + # List what per-language files were created + echo "" + echo "Per-language analysis files created:" + find "$RESULTS_DIR/per_language/" -maxdepth 1 -type f -printf ' %f\n' 2>/dev/null || true +} diff --git a/linux_configuration/scripts/single_use/utils/lib/analyze_repo_scan.sh b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_scan.sh new file mode 100755 index 00000000..b8adc8ab --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_scan.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Output formatting, gitignore-aware file discovery, fast word counting and +# language detection. +# +# Sourced by analyze_repo.sh; split out to keep it under the 250-line cap. +# Sourced rather than run, so it inherits the caller's strict mode and reads +# RESPECT_GITIGNORE, EXCLUDE_DIRS, SCRIPT_DIR and the colour constants that the +# entry script sets above the source line. + +print_header() { + echo "" + echo -e "${BLUE}════════════════════════════════════════════════════════════${NC}" + echo -e "${GREEN} $1${NC}" + echo -e "${BLUE}════════════════════════════════════════════════════════════${NC}" + echo "" +} + +print_subheader() { + echo "" + echo -e "${YELLOW}--- $1 ---${NC}" + echo "" +} + +# Check if we're in a git repository +is_git_repo() { + git rev-parse --is-inside-work-tree &>/dev/null +} + +# Helper function to find files while respecting exclusions +# Usage: find_files "*.c" or find_files "*.py" "*.pyx" +find_files() { + local patterns=("$@") + + if [ "$RESPECT_GITIGNORE" = true ]; then + if is_git_repo; then + # Use git ls-files which respects .gitignore automatically + # This includes tracked files and untracked files not in .gitignore + local git_patterns=() + for pat in "${patterns[@]}"; do + git_patterns+=("$pat") + done + # Get tracked files + untracked (but not ignored) files + { + git ls-files -- "${git_patterns[@]}" 2>/dev/null + git ls-files --others --exclude-standard -- "${git_patterns[@]}" 2>/dev/null + } | sort -u + else + # Not a git repo - fall back to manual exclusion + local find_args=() + for i in "${!patterns[@]}"; do + if [ "$i" -eq 0 ]; then + find_args+=(-name "${patterns[$i]}") + else + find_args+=(-o -name "${patterns[$i]}") + fi + done + find . -type f \( "${find_args[@]}" \) 2>/dev/null | grep -Ev "/($EXCLUDE_DIRS)/" + fi + else + # No filtering - find all files + local find_args=() + for i in "${!patterns[@]}"; do + if [ "$i" -eq 0 ]; then + find_args+=(-name "${patterns[$i]}") + else + find_args+=(-o -name "${patterns[$i]}") + fi + done + find . -type f \( "${find_args[@]}" \) 2>/dev/null + fi +} + +# Count files matching pattern (respecting exclusions) +count_files() { + find_files "$@" | wc -l +} + +# Fast word counting: 'counts' (Rust) if available, else the Python fallback. +fast_count() { + local top_n="${1:-50}" + if command -v counts &>/dev/null; then + counts 2>/dev/null | head -n "$((top_n + 1))" | tail -n "$top_n" + else + python3 "$SCRIPT_DIR/fast_count.py" "$top_n" + fi +} diff --git a/linux_configuration/scripts/single_use/utils/lib/analyze_repo_separate.sh b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_separate.sh new file mode 100755 index 00000000..df366dd6 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_separate.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# Language detection, and per-language comment/code separation: fills a map of +# language -> temp file holding that language's comment-stripped source, plus +# one file of all extracted comments. +# +# Sourced by analyze_repo.sh; split out to keep it under the 250-line cap. +# Sourced rather than run, so it inherits the caller's strict mode and reads +# RESPECT_GITIGNORE, RESULTS_DIR and the colour constants. +# +# detect_languages lives here rather than beside the other file-discovery +# helpers because the HAS_* flags it sets are read only by +# separate_code_and_comments below. Keeping definition and use in one file is +# what lets shellcheck see them as used. + +# separate_code_and_comments +# The map is filled by nameref rather than left in a global, so no mutable +# state crosses the file boundary into the reporting pass. +# Populates LANG_FILES and the HAS_* family from the files present in the cwd. +detect_languages() { + print_subheader "Detecting languages in repository..." + + if [ "$RESPECT_GITIGNORE" = true ]; then + if is_git_repo; then + echo -e "${YELLOW}Note: Respecting .gitignore (excludes node_modules, build outputs, etc.)${NC}" + else + echo -e "${YELLOW}Note: Excluding common directories (node_modules, .git, vendor, etc.)${NC}" + fi + echo " Use --no-ignore to include everything." + echo "" + fi + + # Count files by extension to detect primary languages (using helper). + # LANG_FILES is read only here; the HAS_* flags below are what the callers + # downstream consume, so they stay global. + local lang count + local -A LANG_FILES + LANG_FILES[c]=$(count_files "*.c") + LANG_FILES[cpp]=$(count_files "*.cpp" "*.cc" "*.cxx") + LANG_FILES[h]=$(count_files "*.h" "*.hpp" "*.hxx") + LANG_FILES[python]=$(count_files "*.py") + LANG_FILES[javascript]=$(count_files "*.js") + LANG_FILES[typescript]=$(count_files "*.ts" "*.tsx") + LANG_FILES[java]=$(count_files "*.java") + LANG_FILES[go]=$(count_files "*.go") + LANG_FILES[rust]=$(count_files "*.rs") + LANG_FILES[ruby]=$(count_files "*.rb") + LANG_FILES[shell]=$(count_files "*.sh" "*.bash") + + echo "Files found by language:" + for lang in c cpp h python javascript typescript java go rust ruby shell; do + count=${LANG_FILES[$lang]} + [ "$count" -gt 0 ] && echo " $lang: $count files" + done + + # Determine which language families are present + HAS_C_FAMILY=false + HAS_PYTHON=false + HAS_JS_FAMILY=false + HAS_SHELL=false + HAS_RUBY=false + HAS_GO=false + HAS_RUST=false + HAS_JAVA=false + + ((LANG_FILES[c] + LANG_FILES[cpp] + LANG_FILES[h] > 0)) && HAS_C_FAMILY=true + ((LANG_FILES[python] > 0)) && HAS_PYTHON=true + ((LANG_FILES[javascript] + LANG_FILES[typescript] > 0)) && HAS_JS_FAMILY=true + ((LANG_FILES[shell] > 0)) && HAS_SHELL=true + ((LANG_FILES[ruby] > 0)) && HAS_RUBY=true + ((LANG_FILES[go] > 0)) && HAS_GO=true + ((LANG_FILES[rust] > 0)) && HAS_RUST=true + ((LANG_FILES[java] > 0)) && HAS_JAVA=true +} + +separate_code_and_comments() { + local -n LANG_CODE_FILES="$1" + local COMMENTS_TEMP="$2" + + # Create per-language output directory + mkdir -p "$RESULTS_DIR/per_language" + + # Process C/C++ files + if $HAS_C_FAMILY; then + echo "Processing C/C++ files..." + LANG_CODE_FILES["c_cpp"]=$(mktemp /tmp/code_c_cpp.XXXXXX.tmp) + find_files "*.c" "*.cpp" "*.cc" "*.cxx" "*.h" "*.hpp" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[c_cpp]}" + + # Extract and strip C-style comments + perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[c_cpp]}" >>"$COMMENTS_TEMP" + perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[c_cpp]}" >"${LANG_CODE_FILES[c_cpp]}.clean" + mv "${LANG_CODE_FILES[c_cpp]}.clean" "${LANG_CODE_FILES[c_cpp]}" + fi + + # Process JavaScript files (separate from TypeScript) + if $HAS_JS_FAMILY; then + echo "Processing JavaScript files..." + LANG_CODE_FILES["javascript"]=$(mktemp /tmp/code_js.XXXXXX.tmp) + find_files "*.js" "*.jsx" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[javascript]}" + + echo "Processing TypeScript files..." + LANG_CODE_FILES["typescript"]=$(mktemp /tmp/code_ts.XXXXXX.tmp) + find_files "*.ts" "*.tsx" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[typescript]}" + + # Extract and strip comments from both + for lang_file in "${LANG_CODE_FILES[javascript]}" "${LANG_CODE_FILES[typescript]}"; do + [ ! -s "$lang_file" ] && continue + perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "$lang_file" >>"$COMMENTS_TEMP" + perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "$lang_file" >"${lang_file}.clean" + mv "${lang_file}.clean" "$lang_file" + done + fi + + # Process Python files + if $HAS_PYTHON; then + echo "Processing Python files..." + LANG_CODE_FILES["python"]=$(mktemp /tmp/code_python.XXXXXX.tmp) + find_files "*.py" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[python]}" + + perl -ne 'if (/^\s*#(.*)/) { print "$1\n"; } elsif (/#(.*)$/) { print "$1\n"; }' "${LANG_CODE_FILES[python]}" >>"$COMMENTS_TEMP" + perl -0777 -ne 'while (/"""(.+?)"""/gs) { print "$1\n"; } while (/'"'"''"'"''"'"'(.+?)'"'"''"'"''"'"'/gs) { print "$1\n"; }' "${LANG_CODE_FILES[python]}" >>"$COMMENTS_TEMP" + perl -pe 's/#.*$//' "${LANG_CODE_FILES[python]}" | perl -0777 -pe 's/""".*?"""//gs; s/'"'"''"'"''"'"'.*?'"'"''"'"''"'"'//gs' >"${LANG_CODE_FILES[python]}.clean" + mv "${LANG_CODE_FILES[python]}.clean" "${LANG_CODE_FILES[python]}" + fi + + # Process Go files + if $HAS_GO; then + echo "Processing Go files..." + LANG_CODE_FILES["go"]=$(mktemp /tmp/code_go.XXXXXX.tmp) + find_files "*.go" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[go]}" + + perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[go]}" >>"$COMMENTS_TEMP" + perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[go]}" >"${LANG_CODE_FILES[go]}.clean" + mv "${LANG_CODE_FILES[go]}.clean" "${LANG_CODE_FILES[go]}" + fi + + # Process Rust files + if $HAS_RUST; then + echo "Processing Rust files..." + LANG_CODE_FILES["rust"]=$(mktemp /tmp/code_rust.XXXXXX.tmp) + find_files "*.rs" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[rust]}" + + perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[rust]}" >>"$COMMENTS_TEMP" + perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[rust]}" >"${LANG_CODE_FILES[rust]}.clean" + mv "${LANG_CODE_FILES[rust]}.clean" "${LANG_CODE_FILES[rust]}" + fi + + # Process Ruby files + if $HAS_RUBY; then + echo "Processing Ruby files..." + LANG_CODE_FILES["ruby"]=$(mktemp /tmp/code_ruby.XXXXXX.tmp) + find_files "*.rb" | head -5000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[ruby]}" + + perl -ne 'if (/#(.*)$/) { print "$1\n"; }' "${LANG_CODE_FILES[ruby]}" >>"$COMMENTS_TEMP" + perl -0777 -ne 'while (/=begin(.+?)=end/gs) { print "$1\n"; }' "${LANG_CODE_FILES[ruby]}" >>"$COMMENTS_TEMP" + perl -pe 's/#.*$//' "${LANG_CODE_FILES[ruby]}" | perl -0777 -pe 's/=begin.*?=end//gs' >"${LANG_CODE_FILES[ruby]}.clean" + mv "${LANG_CODE_FILES[ruby]}.clean" "${LANG_CODE_FILES[ruby]}" + fi + + # Process Shell files + if $HAS_SHELL; then + echo "Processing Shell files..." + LANG_CODE_FILES["shell"]=$(mktemp /tmp/code_shell.XXXXXX.tmp) + find_files "*.sh" "*.bash" | head -5000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[shell]}" + + perl -ne 'if (/^\s*#(.*)/ && !/^#!/) { print "$1\n"; } elsif (/#(.*)$/) { print "$1\n"; }' "${LANG_CODE_FILES[shell]}" >>"$COMMENTS_TEMP" + perl -pe 's/#.*$//' "${LANG_CODE_FILES[shell]}" >"${LANG_CODE_FILES[shell]}.clean" + mv "${LANG_CODE_FILES[shell]}.clean" "${LANG_CODE_FILES[shell]}" + fi + + # Process Java files + if $HAS_JAVA; then + echo "Processing Java files..." + LANG_CODE_FILES["java"]=$(mktemp /tmp/code_java.XXXXXX.tmp) + find_files "*.java" | head -15000 | xargs cat 2>/dev/null >"${LANG_CODE_FILES[java]}" + + perl -0777 -ne 'while (/\/\*(.+?)\*\//gs) { print "$1\n"; } while (/\/\/([^\n]*)/g) { print "$1\n"; }' "${LANG_CODE_FILES[java]}" >>"$COMMENTS_TEMP" + perl -0777 -pe 's|/\*.*?\*/||gs; s|//[^\n]*||g;' "${LANG_CODE_FILES[java]}" >"${LANG_CODE_FILES[java]}.clean" + mv "${LANG_CODE_FILES[java]}.clean" "${LANG_CODE_FILES[java]}" + fi + + local COMMENT_LINES + COMMENT_LINES=$(wc -l <"$COMMENTS_TEMP") + echo "" + echo "Processed languages: ${!LANG_CODE_FILES[*]}" + echo "Total comment lines: $COMMENT_LINES" +} diff --git a/linux_configuration/scripts/single_use/utils/lib/analyze_repo_symbols.sh b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_symbols.sh new file mode 100755 index 00000000..4fbcb8c0 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_symbols.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# ctags symbol extraction, the cscope database, and the clang AST sample. +# +# Sourced by analyze_repo.sh; split out to keep it under the 250-line cap. +# Sourced rather than run, so it inherits the caller's strict mode and reads +# RESULTS_DIR, TOP_N, RESPECT_GITIGNORE, EXCLUDE_DIRS and the colour constants. + +analyze_symbols() { + print_subheader "Generating tags (this may take a while)..." + + # Generate tags for different kinds + ctags -R --languages=C,C++ --c-kinds=+fp --fields=+lK -f "$RESULTS_DIR/tags" . 2>/dev/null || true + + if [ -f "$RESULTS_DIR/tags" ]; then + TOTAL_TAGS=$(grep -ac '^[^!]' "$RESULTS_DIR/tags" 2>/dev/null || echo "0") + echo "Total symbols found: $TOTAL_TAGS" + + print_subheader "Most Common Symbol Names" + # Fast: use cut + counts instead of awk + sort | uniq + # -a flag treats tags file as text (may contain binary-like patterns) + grep -a '^[^!]' "$RESULTS_DIR/tags" | cut -f1 | fast_count "$TOP_N" | + tee "$RESULTS_DIR/ctags_symbols.txt" + + print_subheader "Symbol Types Distribution" + # Fast: extract single-letter kind code after ;" and count + grep -aoP ';"\t\K[a-z]' "$RESULTS_DIR/tags" 2>/dev/null | fast_count 20 | while read -r count kind; do + case $kind in + f) echo "$count functions" ;; + v) echo "$count variables" ;; + s) echo "$count structs" ;; + t) echo "$count typedefs" ;; + e) echo "$count enum values" ;; + g) echo "$count enums" ;; + m) echo "$count struct/union members" ;; + d) echo "$count macro definitions" ;; + p) echo "$count function prototypes" ;; + u) echo "$count unions" ;; + c) echo "$count classes" ;; + n) echo "$count namespaces" ;; + *) echo "$count kind=$kind" ;; + esac + done | tee "$RESULTS_DIR/ctags_kinds.txt" + fi + +} + +analyze_cscope() { + print_subheader "Building cscope database..." + + # Find all C source files (respecting .gitignore if available) + if [ "$RESPECT_GITIGNORE" = true ] && is_git_repo; then + { + git ls-files -- '*.c' '*.h' 2>/dev/null + git ls-files --others --exclude-standard -- '*.c' '*.h' 2>/dev/null + } | sort -u >"$RESULTS_DIR/cscope.files" + elif [ "$RESPECT_GITIGNORE" = true ]; then + find . \( -name "*.c" -o -name "*.h" \) -type f 2>/dev/null | grep -Ev "/($EXCLUDE_DIRS)/" >"$RESULTS_DIR/cscope.files" + else + find . \( -name "*.c" -o -name "*.h" \) -type f >"$RESULTS_DIR/cscope.files" 2>/dev/null + fi + FILE_COUNT=$(wc -l <"$RESULTS_DIR/cscope.files") + echo "Found $FILE_COUNT source files" + + # Build cscope database (can take a while for large repos) + echo "Building database (this may take several minutes for Linux kernel)..." + cscope -b -q -i "$RESULTS_DIR/cscope.files" -f "$RESULTS_DIR/cscope.out" 2>/dev/null || true + + if [ -f "$RESULTS_DIR/cscope.out" ]; then + echo "Database built successfully" + echo "Database size: $(du -sh "$RESULTS_DIR/cscope.out" | cut -f1)" + + print_subheader "Example: Finding callers of 'printk' function" + cscope -d -f "$RESULTS_DIR/cscope.out" -L -3 printk 2>/dev/null | head -20 || echo "No results" + + print_subheader "Example: Finding definition of 'struct file'" + cscope -d -f "$RESULTS_DIR/cscope.out" -L -1 "struct file" 2>/dev/null | head -10 || echo "No results" + fi + +} + +analyze_clang_ast() { + print_subheader "Analyzing a sample file with clang AST dump" + + # Find a simple C file to analyze (respecting .gitignore) + if [ "$RESPECT_GITIGNORE" = true ] && is_git_repo; then + SAMPLE_FILE=$(git ls-files -- '*.c' 2>/dev/null | head -20 | while read -r f; do + [ -f "$f" ] && [ "$(stat -c%s "$f" 2>/dev/null || echo 999999)" -lt 51200 ] && echo "$f" + done | head -1) + elif [ "$RESPECT_GITIGNORE" = true ]; then + SAMPLE_FILE=$(find . -name "*.c" -size -50k -type f 2>/dev/null | grep -Ev "/($EXCLUDE_DIRS)/" | head -1) + else + SAMPLE_FILE=$(find . -name "*.c" -size -50k 2>/dev/null | head -1) + fi + + if [ -n "$SAMPLE_FILE" ]; then + echo "Sample file: $SAMPLE_FILE" + echo "" + echo "Function declarations in this file:" + clang -Xclang -ast-dump -fsyntax-only "$SAMPLE_FILE" 2>/dev/null | + grep -E "FunctionDecl.*<.*>" | + head -20 | + sed 's/.*FunctionDecl.*<[^>]*> / /' | + tee "$RESULTS_DIR/clang_sample_functions.txt" || echo "Analysis failed (missing headers)" + fi + + print_subheader "Note: Full clang analysis requires compile_commands.json" + echo "For proper AST analysis of the Linux kernel, you need to:" + echo " 1. Configure the kernel: make defconfig" + echo " 2. Generate compile_commands.json: make compile_commands.json" + echo " 3. Use clang-query or clang-check with the database" + +} diff --git a/linux_configuration/scripts/single_use/utils/lib/analyze_repo_tools.sh b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_tools.sh new file mode 100755 index 00000000..38fc84b4 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/analyze_repo_tools.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Package-manager detection and installation of the analysis toolchain +# (ctags, cscope, clang, ugrep, tokei, scc, counts). +# +# Sourced by analyze_repo.sh; split out to keep it under the 250-line cap. +# Sourced rather than run, so it inherits the caller's strict mode and the +# colour constants defined above the source line. + +install_missing_tools() { + local MISSING_TOOLS=() + local MISSING_AUR=() + + # Check for required tools + command -v git &>/dev/null || MISSING_TOOLS+=("git") + command -v ctags &>/dev/null || MISSING_TOOLS+=("ctags") + command -v cscope &>/dev/null || MISSING_TOOLS+=("cscope") + command -v clang &>/dev/null || MISSING_TOOLS+=("clang") + command -v ugrep &>/dev/null || MISSING_TOOLS+=("ugrep") + + # Check for AUR tools + command -v tokei &>/dev/null || MISSING_AUR+=("tokei") + command -v scc &>/dev/null || MISSING_AUR+=("scc") + + # Check for Rust 'counts' tool (install via cargo if missing) + if ! command -v counts &>/dev/null; then + if command -v cargo &>/dev/null; then + echo "Installing 'counts' via cargo (fast word counter)..." + cargo install counts 2>/dev/null || echo "Warning: counts install failed, will use Python fallback" + fi + fi + + # If nothing is missing, return + if [ ${#MISSING_TOOLS[@]} -eq 0 ] && [ ${#MISSING_AUR[@]} -eq 0 ]; then + echo -e "${GREEN}All required tools are installed.${NC}" + return 0 + fi + + echo -e "${YELLOW}Missing tools detected. Installing...${NC}" + + # Detect package manager + if command -v pacman &>/dev/null; then + # Arch Linux + if [ ${#MISSING_TOOLS[@]} -gt 0 ]; then + echo "Installing from official repos: ${MISSING_TOOLS[*]}" + sudo pacman -S --needed --noconfirm "${MISSING_TOOLS[@]}" + fi + + if [ ${#MISSING_AUR[@]} -gt 0 ]; then + # Find or install AUR helper + if command -v yay &>/dev/null; then + AUR_HELPER="yay" + elif command -v paru &>/dev/null; then + AUR_HELPER="paru" + else + echo "No AUR helper found. Installing yay..." + sudo pacman -S --needed --noconfirm base-devel git + TEMP_DIR=$(mktemp -d) + git clone https://aur.archlinux.org/yay.git "$TEMP_DIR/yay" + (cd "$TEMP_DIR/yay" && makepkg -si --noconfirm) + rm -rf "$TEMP_DIR" + AUR_HELPER="yay" + fi + + echo "Installing from AUR: ${MISSING_AUR[*]}" + $AUR_HELPER -S --needed --noconfirm "${MISSING_AUR[@]}" + fi + + elif command -v apt-get &>/dev/null; then + # Debian/Ubuntu + echo "Installing tools via apt..." + sudo apt-get update + + # Map tool names to package names + APT_PACKAGES=() + for tool in "${MISSING_TOOLS[@]}"; do + case $tool in + ctags) APT_PACKAGES+=("universal-ctags") ;; + ugrep) APT_PACKAGES+=("ugrep") ;; + *) APT_PACKAGES+=("$tool") ;; + esac + done + + [ ${#APT_PACKAGES[@]} -gt 0 ] && sudo apt-get install -y "${APT_PACKAGES[@]}" + + # Install tokei/scc via cargo or snap + for aur_tool in "${MISSING_AUR[@]}"; do + if command -v cargo &>/dev/null; then + echo "Installing $aur_tool via cargo..." + cargo install "$aur_tool" + elif command -v snap &>/dev/null; then + echo "Installing $aur_tool via snap..." + sudo snap install "$aur_tool" + else + echo -e "${YELLOW}Warning: Cannot install $aur_tool. Install cargo or snap first.${NC}" + fi + done + + elif command -v dnf &>/dev/null; then + # Fedora + echo "Installing tools via dnf..." + sudo dnf install -y "${MISSING_TOOLS[@]}" "${MISSING_AUR[@]}" 2>/dev/null || { + # tokei/scc might need cargo + for aur_tool in "${MISSING_AUR[@]}"; do + if command -v cargo &>/dev/null; then + cargo install "$aur_tool" + fi + done + } + + elif command -v brew &>/dev/null; then + # macOS with Homebrew + echo "Installing tools via brew..." + ALL_TOOLS=("${MISSING_TOOLS[@]}" "${MISSING_AUR[@]}") + brew install "${ALL_TOOLS[@]}" + + else + echo -e "${RED}Unknown package manager. Please install these tools manually:${NC}" + echo " Official: ${MISSING_TOOLS[*]}" + echo " Additional: ${MISSING_AUR[*]}" + exit 1 + fi + + echo -e "${GREEN}Tool installation complete.${NC}" +}