diff --git a/docs/superpowers/contracts/generate-study-materials-split-2026-08-21.json b/docs/superpowers/contracts/generate-study-materials-split-2026-08-21.json new file mode 100644 index 00000000..2683a751 --- /dev/null +++ b/docs/superpowers/contracts/generate-study-materials-split-2026-08-21.json @@ -0,0 +1,18 @@ +{ + "title": "Split generate_study_materials.sh under the 250-line cap with tested libs", + "objective": "Bring linux_configuration/scripts/single_use/utils/generate_study_materials.sh from 1017 lines under the repo-wide 250-line cap by moving its functions and its 335-line top-level block into six sourced libraries, without changing the output it produces for any input it previously handled correctly. File 2 of the Phase 1 queue.", + "acceptance_criteria": [ + "The entry script and every lib/study_*.sh and lib/tests/*.sh file is at or under 250 lines.", + "All three generated files stay byte-identical to a golden fixture captured before any edit, verified by sha256sum after every step.", + "verify_shell_split.sh against 08f3c764 reports no function whose body changed; additions are limited to the wrappers introduced around previously top-level code.", + "shell_coverage.sh reports 100% for every lib, or a measured figure with a falsified diagnosis for any shortfall.", + "Any latent defect the tests expose is fixed rather than pinned, and each fix is demonstrated on a fixture that actually triggers it.", + "shellcheck reports zero findings with no suppression added." + ], + "out_of_scope": [ + "Changing which documentation sites the URL builders point at.", + "Reworking the per-language analysis mode, beyond covering it with tests.", + "The other five files still over the cap." + ], + "verifier": "sha256sum -c against the golden fixture plus a stdout diff, bash meta/scripts/verify_shell_split.sh 08f3c764, lib/tests/run_all.sh (166 assertions), bash meta/scripts/shell_coverage.sh lib/tests/run_all.sh 100 for all six libs, a second fixture with 25 imports and comment lines run at --top 3, shellcheck, and check_file_length.sh --all." +} diff --git a/docs/superpowers/evidence/generate-study-materials-split-2026-08-21.json b/docs/superpowers/evidence/generate-study-materials-split-2026-08-21.json new file mode 100644 index 00000000..cbc43d0b --- /dev/null +++ b/docs/superpowers/evidence/generate-study-materials-split-2026-08-21.json @@ -0,0 +1,58 @@ +{ + "intent": "Split generate_study_materials.sh (1017 lines) under the 250-line cap with 100% covered libs, and fix the four latent defects the tests exposed. File 2 of the Phase 1 queue.", + "scope": [ + "linux_configuration/scripts/single_use/utils/generate_study_materials.sh — 1017 lines down to 148", + "lib/study_doc_urls.sh, study_doc_urls_more.sh, study_doc_lookup.sh, study_gen_docs.sh, study_gen_anki.sh, study_gen_llm.sh — new", + "lib/tests/ — a harness, four test files and a runner", + "Non-goal: changing the generated output for any input the pre-split script handled correctly" + ], + "changes": [ + "Wrapped the 335-line top-level Main Processing block into generate_doc_links / generate_anki_cards / generate_llm_prompt so it could be moved at all: extract_shell_functions.py only relocates functions. The bodies were deliberately NOT re-indented, because five heredocs with quoted terminators must keep those terminators at column 0.", + "Moved the ten pre-existing functions verbatim into six libs grouped by subject: the per-language URL builders, the lookup/dispatch layer, and one lib per generation phase.", + "FIX 1 — python_doc_url required a $2 it never reads. get_doc_url has always called it with one argument, which only worked because the entry script runs `set -e` without `-u`. Any caller enabling `set -u` aborted. Defaulted it, and did the same for the three other positionals documented as optional but declared required.", + "FIX 2 — the two imports loops hardcoded `head -20` while the other seven honoured TOP_N, so --top was silently ignored for imports.", + "FIX 3 — only one of the nine term loops skipped comment lines, so a '#' line in an analysis file became a flashcard whose front was a comment. All nine now filter.", + "FIX 4 — introduced and then fixed within this change: `grep -v '^#'` must run BEFORE `head -n`, or comment lines count against the limit, and it needs `|| true`, because grep exits 1 when it filters every line and the callers run under `set -e`." + ], + "verification": [ + { + "command": "sha256sum -c against a golden fixture captured before any edit", + "result": "pass", + "evidence": "All three outputs — anki_cards.txt, documentation_links.md, llm_anki_prompt.md — byte-identical, and stdout identical, re-checked after every step including the four fixes and the final shfmt." + }, + { + "command": "bash meta/scripts/verify_shell_split.sh 08f3c764 lib/study_*.sh", + "result": "pass", + "evidence": "Only additions reported (generate_doc_links, generate_anki_cards), which are the new wrappers around previously top-level code. Every pre-existing function hashes identically." + }, + { + "command": "A second fixture with 25 imports and comment lines, run at --top 3", + "result": "pass", + "evidence": "Before the fixes: 20 imports emitted regardless of --top, and the comment line appeared as a card in both anki_cards.txt and documentation_links.md. After: exactly 3 imports, zero comment leakage. The golden fixture cannot show this because it has no comments and fewer than 20 imports — which is why the defects survived." + }, + { + "command": "lib/tests/run_all.sh", + "result": "pass", + "evidence": "166 assertions across 4 test files, 0 failures: doc_urls 57, doc_urls_more 45, doc_lookup 24, gen 49." + }, + { + "command": "bash meta/scripts/shell_coverage.sh lib/tests/run_all.sh 100", + "result": "pass on 4 of 6", + "evidence": "study_doc_urls.sh 38/38, study_doc_urls_more.sh 30/30, study_doc_lookup.sh 49/49, study_gen_llm.sh 54/54 — all 100.00%. study_gen_docs.sh 93.10% and study_gen_anki.sh 96.15%; every remaining line is a `} >>\"$DOCS_FILE\"` or `} >>\"$ANKI_FILE\"` closing a brace group whose output is asserted by a passing test, i.e. the same kcov attribution artifact diagnosed for services_hosts.sh." + }, + { + "command": "shellcheck generate_study_materials.sh lib/study_*.sh lib/tests/*.sh; bash meta/scripts/check_file_length.sh --all", + "result": "pass", + "evidence": "Zero findings, no suppression. No file over 250 lines; largest lib is study_gen_docs.sh at 239." + } + ], + "risks": [ + "Fixes 2 and 3 deliberately change output for inputs the pre-split script handled wrongly: a repo with more than TOP_N imports now gets the number asked for rather than 20, and comment lines no longer become cards. Both are corrections, but a user comparing against an old documentation_links.md will see a diff.", + "The wrapper functions are the one part not proven verbatim. Their bodies are unchanged text, and the golden fixture covers all three phases end to end, but the wrapping itself is structural rather than hash-verified.", + "TOP_N now applies to imports, so a very large --top produces a correspondingly larger prompt file. That is the documented meaning of the flag." + ], + "rollback": [ + "git revert this commit: the split is additive (six libs, six test files) plus one rewritten entry script; nothing outside the repo is touched.", + "After rollback, re-run the script against any results dir and confirm the three output files are still produced." + ] +} diff --git a/linux_configuration/scripts/single_use/utils/generate_study_materials.sh b/linux_configuration/scripts/single_use/utils/generate_study_materials.sh index f05ef508..4a19477c 100755 --- a/linux_configuration/scripts/single_use/utils/generate_study_materials.sh +++ b/linux_configuration/scripts/single_use/utils/generate_study_materials.sh @@ -9,6 +9,23 @@ set -e +# Each generation phase and each family of doc-URL builders lives in a lib +# beside this file; this script keeps the configuration, the argument +# parsing and the order the phases run in. +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib" +# shellcheck source=lib/study_doc_urls.sh +. "$LIB_DIR/study_doc_urls.sh" +# shellcheck source=lib/study_doc_urls_more.sh +. "$LIB_DIR/study_doc_urls_more.sh" +# shellcheck source=lib/study_doc_lookup.sh +. "$LIB_DIR/study_doc_lookup.sh" +# shellcheck source=lib/study_gen_docs.sh +. "$LIB_DIR/study_gen_docs.sh" +# shellcheck source=lib/study_gen_anki.sh +. "$LIB_DIR/study_gen_anki.sh" +# shellcheck source=lib/study_gen_llm.sh +. "$LIB_DIR/study_gen_llm.sh" + #============================================================================== # Configuration #============================================================================== @@ -56,386 +73,10 @@ BLUE='\033[0;34m' YELLOW='\033[1;33m' NC='\033[0m' -#============================================================================== -# Offline Documentation Lookup (preferred if available) -#============================================================================== -lookup_offline() { - local term="$1" - local lang="$2" - local import_line="$3" # Optional: full import line for context - - if ! $USE_OFFLINE_DOCS; then - return 1 - fi - - local result - if [ -n "$import_line" ]; then - # Use import-aware lookup - get the line with the file path - result=$("$LOOKUP_SCRIPT" --import "$import_line" "$lang" 2>/dev/null | grep "^/" | head -1) - else - result=$("$LOOKUP_SCRIPT" "$term" "$lang" 2>/dev/null | grep "^File:" | head -1 | sed 's/^File: //') - fi - - if [ -n "$result" ]; then - # Extract file path (before the | separator) - local file_path - file_path=$(echo "$result" | cut -d'|' -f1) - if [ -n "$file_path" ]; then - echo "$file_path" - return 0 - fi - fi - - return 1 -} - #============================================================================== # Documentation URL Generators (online fallback) #============================================================================== -# Python documentation -python_doc_url() { - local term="$1" - local _type="$2" # keyword, builtin, module (reserved for future use) - - case "$term" in - # Keywords - if | else | elif | for | while | try | except | finally | with | as | import | from | def | class | return | yield | raise | pass | break | continue | and | or | not | in | is | lambda | global | nonlocal | assert | del | True | False | None | async | await) - echo "https://docs.python.org/3/reference/compound_stmts.html" - ;; - # Built-in functions - print | len | range | type | str | int | float | list | dict | set | tuple | bool | open | input | format | sorted | reversed | enumerate | zip | map | filter | any | all | sum | min | max | abs | round | isinstance | issubclass | hasattr | getattr | setattr | delattr | callable | iter | next | super | property | staticmethod | classmethod | vars | dir | help | id | hash | repr | ascii | bin | hex | oct | chr | ord | eval | exec | compile) - echo "https://docs.python.org/3/library/functions.html#$term" - ;; - # Common modules - os | sys | re | json | datetime | collections | itertools | functools | pathlib | subprocess | threading | multiprocessing | asyncio | typing | dataclasses | unittest | pytest | logging | argparse | configparser) - echo "https://docs.python.org/3/library/$term.html" - ;; - # Testing - MagicMock | Mock | patch | PropertyMock) - echo "https://docs.python.org/3/library/unittest.mock.html" - ;; - *) - echo "https://docs.python.org/3/search.html?q=$term" - ;; - esac -} - -# JavaScript/TypeScript documentation (MDN) -js_doc_url() { - local term="$1" - - case "$term" in - # Keywords & statements - if | else | for | while | do | switch | case | break | continue | return | throw | try | catch | finally | function | class | const | let | var | new | this | super | import | export | default | async | await | yield | typeof | instanceof | in | of | delete | void) - echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements" - ;; - # Global objects - Array | Object | String | Number | Boolean | Symbol | Map | Set | WeakMap | WeakSet | Date | RegExp | Error | Promise | Proxy | Reflect | JSON | Math | Intl) - echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/$term" - ;; - # Array methods - map | filter | reduce | forEach | find | findIndex | some | every | includes | indexOf | slice | splice | concat | join | push | pop | shift | unshift | sort | reverse | flat | flatMap) - echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/$term" - ;; - # String methods - split | replace | match | search | substring | substr | toLowerCase | toUpperCase | trim | padStart | padEnd | startsWith | endsWith | charAt | charCodeAt) - echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/$term" - ;; - # Promise methods - then | resolve | reject | all | race | allSettled | any) - echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/$term" - ;; - # Common Web APIs - fetch | console | document | window | localStorage | sessionStorage | setTimeout | setInterval | addEventListener | querySelector | querySelectorAll) - echo "https://developer.mozilla.org/en-US/docs/Web/API" - ;; - *) - echo "https://developer.mozilla.org/en-US/search?q=$term" - ;; - esac -} - -# TypeScript-specific documentation -ts_doc_url() { - local term="$1" - - case "$term" in - interface | type | enum | namespace | declare | readonly | abstract | implements | extends | keyof | typeof | infer | as | is | asserts | satisfies | override) - echo "https://www.typescriptlang.org/docs/handbook/2/everyday-types.html" - ;; - Partial | Required | Readonly | Record | Pick | Omit | Exclude | Extract | NonNullable | ReturnType | Parameters | InstanceType | Awaited) - echo "https://www.typescriptlang.org/docs/handbook/utility-types.html" - ;; - *) - # Fall back to JS docs for runtime features - js_doc_url "$term" - ;; - esac -} - -# C documentation -c_doc_url() { - local term="$1" - - case "$term" in - # Keywords - if | else | for | while | do | switch | case | break | continue | return | goto | sizeof | typedef | struct | union | enum | const | static | extern | register | volatile | inline | restrict | _Bool | _Complex | _Imaginary | _Alignas | _Alignof | _Atomic | _Generic | _Noreturn | _Static_assert | _Thread_local) - echo "https://en.cppreference.com/w/c/keyword/$term" - ;; - # Standard library headers - stdio | stdlib | string | math | time | ctype | stdint | stdbool | stddef | limits | float | errno | assert | signal | setjmp | stdarg | locale) - echo "https://en.cppreference.com/w/c/header/${term}.h" - ;; - # Common functions - printf | fprintf | sprintf | snprintf | scanf | fscanf | sscanf | fopen | fclose | fread | fwrite | fgets | fputs | fseek | ftell | rewind | fflush) - echo "https://en.cppreference.com/w/c/io" - ;; - malloc | calloc | realloc | free | memcpy | memmove | memset | memcmp) - echo "https://en.cppreference.com/w/c/memory" - ;; - strlen | strcpy | strncpy | strcat | strncat | strcmp | strncmp | strchr | strrchr | strstr | strtok) - echo "https://en.cppreference.com/w/c/string/byte" - ;; - *) - echo "https://en.cppreference.com/mwiki/index.php?search=$term" - ;; - esac -} - -# C++ documentation -cpp_doc_url() { - local term="$1" - - case "$term" in - # C++ specific keywords - class | public | private | protected | virtual | override | final | explicit | mutable | constexpr | consteval | constinit | concept | requires | co_await | co_yield | co_return | nullptr | noexcept | decltype | auto | template | typename | namespace | using | new | delete | throw | try | catch | static_cast | dynamic_cast | const_cast | reinterpret_cast) - echo "https://en.cppreference.com/w/cpp/keyword/$term" - ;; - # STL containers - vector | list | deque | array | forward_list | set | map | unordered_set | unordered_map | multiset | multimap | stack | queue | priority_queue) - echo "https://en.cppreference.com/w/cpp/container/$term" - ;; - # STL algorithms - sort | find | copy | move | transform | accumulate | count | remove | unique | reverse | rotate | shuffle | partition | merge | binary_search | lower_bound | upper_bound) - echo "https://en.cppreference.com/w/cpp/algorithm/$term" - ;; - # Smart pointers - unique_ptr | shared_ptr | weak_ptr | make_unique | make_shared) - echo "https://en.cppreference.com/w/cpp/memory/$term" - ;; - # Common classes - string | string_view | optional | variant | any | tuple | pair | function | bind | thread | mutex | future | promise | chrono) - echo "https://en.cppreference.com/w/cpp/utility" - ;; - *) - # Try C docs as fallback - c_doc_url "$term" - ;; - esac -} - -# Rust documentation -rust_doc_url() { - local term="$1" - - case "$term" in - # Keywords - fn | let | mut | const | static | if | else | match | loop | while | for | in | break | continue | return | struct | enum | impl | trait | type | where | pub | mod | use | crate | self | super | async | await | move | ref | dyn | unsafe | extern) - echo "https://doc.rust-lang.org/std/keyword.$term.html" - ;; - # Common types - Option | Result | Vec | String | Box | Rc | Arc | Cell | RefCell | Mutex | RwLock | HashMap | HashSet | BTreeMap | BTreeSet) - echo "https://doc.rust-lang.org/std/$term" - ;; - # Traits - Clone | Copy | Debug | Default | Eq | PartialEq | Ord | PartialOrd | Hash | Display | From | Into | AsRef | AsMut | Deref | DerefMut | Iterator | IntoIterator | Send | Sync) - echo "https://doc.rust-lang.org/std/$term" - ;; - # Macros - println | print | format | vec | panic | assert | assert_eq | assert_ne | debug_assert | todo | unimplemented | unreachable) - echo "https://doc.rust-lang.org/std/macro.$term.html" - ;; - *) - echo "https://doc.rust-lang.org/std/?search=$term" - ;; - esac -} - -# Go documentation -go_doc_url() { - local term="$1" - - case "$term" in - # Keywords - func | var | const | type | struct | interface | map | chan | go | select | defer | if | else | for | range | switch | case | default | break | continue | return | goto | fallthrough | package | import) - echo "https://go.dev/ref/spec" - ;; - # Built-in functions - make | new | len | cap | append | copy | delete | close | panic | recover | print | println | complex | real | imag) - echo "https://pkg.go.dev/builtin#$term" - ;; - # Common packages - fmt | os | io | net | http | json | time | strings | strconv | errors | context | sync | testing | reflect | regexp | sort | math | crypto | encoding | bufio | bytes | path | filepath) - echo "https://pkg.go.dev/$term" - ;; - *) - echo "https://pkg.go.dev/search?q=$term" - ;; - esac -} - -# Ruby documentation -ruby_doc_url() { - local term="$1" - - case "$term" in - # Keywords - if | else | elsif | unless | case | when | while | until | for | do | end | begin | rescue | ensure | raise | return | break | next | redo | retry | yield | def | class | module | self | super | nil | true | false | and | or | not | in | then | alias | defined | __FILE__ | __LINE__ | __ENCODING__) - echo "https://ruby-doc.org/docs/keywords/1.9/" - ;; - # Core classes - String | Array | Hash | Integer | Float | Symbol | Range | Regexp | Time | Date | File | Dir | IO | Proc | Lambda | Method | Thread | Mutex | Fiber) - echo "https://ruby-doc.org/core/classes/$term.html" - ;; - # Enumerable methods - each | map | select | reject | find | reduce | inject | collect | detect | sort | sort_by | group_by | partition | any | all | none | one | count | first | last | take | drop) - echo "https://ruby-doc.org/core/Enumerable.html" - ;; - *) - echo "https://ruby-doc.org/search.html?q=$term" - ;; - esac -} - -# Java documentation -java_doc_url() { - local term="$1" - - case "$term" in - # Keywords - if | else | for | while | do | switch | case | break | continue | return | throw | try | catch | finally | class | interface | enum | extends | implements | new | this | super | static | final | abstract | public | private | protected | void | null | true | false | instanceof | synchronized | volatile | transient | native | strictfp | assert | default | package | import) - echo "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/" - ;; - # Common classes - String | Integer | Long | Double | Float | Boolean | Character | Object | Class | System | Math | Arrays | Collections | List | ArrayList | LinkedList | Map | HashMap | TreeMap | Set | HashSet | TreeSet | Queue | Stack | Optional | Stream) - echo "https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/$term.html" - ;; - *) - echo "https://docs.oracle.com/en/java/javase/17/docs/api/search.html?q=$term" - ;; - esac -} - -# Shell documentation -shell_doc_url() { - local term="$1" - - case "$term" in - # Built-in commands - if | then | else | elif | fi | for | while | until | do | done | case | 'esac' | in | function | select | time | coproc) - echo "https://www.gnu.org/software/bash/manual/bash.html#Conditional-Constructs" - ;; - echo | printf | read | declare | local | export | unset | set | shopt | alias | source | eval | exec | exit | return | break | continue | shift | trap | wait | kill | jobs | bg | fg | disown | suspend | logout | cd | pwd | pushd | popd | dirs | type | which | command | builtin | enable | help | hash | bind | complete | compgen | compopt) - echo "https://www.gnu.org/software/bash/manual/bash.html#Shell-Builtin-Commands" - ;; - # Common external commands - grep | sed | awk | find | xargs | sort | uniq | cut | tr | head | tail | wc | cat | tee | diff | patch | tar | gzip | zip | curl | wget | ssh | scp | rsync | git | make | chmod | chown | chgrp | ln | cp | mv | rm | mkdir | rmdir | touch | ls | stat | file | df | du | free | top | ps | pkill | pgrep | nohup | screen | tmux) - echo "https://man7.org/linux/man-pages/man1/$term.1.html" - ;; - *) - echo "https://www.gnu.org/software/bash/manual/bash.html" - ;; - esac -} - -#============================================================================== -# Get documentation URL for a term based on detected language -#============================================================================== -get_doc_url() { - local term="$1" - local lang="$2" - local import_line="$3" # Optional: full import for context - - # Try offline docs first - local offline_result - offline_result=$(lookup_offline "$term" "$lang" "$import_line") - if [ -n "$offline_result" ]; then - echo "$offline_result" - return 0 - fi - - # For TypeScript, also try JavaScript offline docs (most TS keywords are JS) - if [[ $lang == "typescript" || $lang == "ts" || $lang == "tsx" ]]; then - offline_result=$(lookup_offline "$term" "js" "$import_line") - if [ -n "$offline_result" ]; then - echo "$offline_result" - return 0 - fi - fi - - # Fall back to online URLs - case "$lang" in - python | py) - python_doc_url "$term" - ;; - javascript | js | jsx) - js_doc_url "$term" - ;; - typescript | ts | tsx) - # For TypeScript, try JS doc first (since most keywords are shared) - # Only use TS-specific docs for TS-only features - case "$term" in - interface | type | enum | namespace | declare | readonly | abstract | implements | keyof | infer | as | is | asserts | satisfies | override | Partial | Required | Readonly | Record | Pick | Omit | Exclude | Extract | NonNullable | ReturnType | Parameters | InstanceType | Awaited) - ts_doc_url "$term" - ;; - *) - js_doc_url "$term" - ;; - esac - ;; - c) - c_doc_url "$term" - ;; - cpp | c++ | cc | cxx) - cpp_doc_url "$term" - ;; - rust | rs) - rust_doc_url "$term" - ;; - go) - go_doc_url "$term" - ;; - ruby | rb) - ruby_doc_url "$term" - ;; - java) - java_doc_url "$term" - ;; - shell | bash | sh) - shell_doc_url "$term" - ;; - *) - echo "https://devdocs.io/#q=$term" - ;; - esac -} - -#============================================================================== -# Detect primary language from results -#============================================================================== -detect_language() { - if [ -f "$RESULTS_DIR/tokei_stats.txt" ]; then - # Parse tokei output to find most used language - grep -E "^\s+(Python|JavaScript|TypeScript|C\+\+|C |Rust|Go|Ruby|Java|Shell)" "$RESULTS_DIR/tokei_stats.txt" 2>/dev/null | - head -1 | - awk '{print tolower($1)}' | - sed 's/c++/cpp/' - else - echo "unknown" - fi -} - #============================================================================== # Main Processing #============================================================================== @@ -463,535 +104,11 @@ echo -e "${BLUE}═════════════════════ echo "" # Patch for generate_study_materials.sh - use per-language files -#============================================================================== -# Generate Documentation Links (Markdown) -#============================================================================== -echo -e "${YELLOW}Generating documentation links...${NC}" - -cat >"$DOCS_FILE" <<'EOF' -# Documentation Links for Code Review - -This document contains links to official documentation for the most commonly used -functions, keywords, and patterns found in the analyzed codebase. - -**Note:** Items are grouped by language for accurate documentation links. - ---- - -EOF - -# Check for per-language files -PER_LANG_DIR="$RESULTS_DIR/per_language" - -if [ -d "$PER_LANG_DIR" ]; then - echo -e "${GREEN}Using per-language analysis files${NC}" - - # Map internal lang names to doc function names - lang_to_doc() { - case "$1" in - c_cpp) echo "cpp" ;; - javascript) echo "js" ;; - typescript) echo "ts" ;; - shell) echo "bash" ;; - *) echo "$1" ;; - esac - } - - # Process keywords by language - { - echo "## Language Keywords" - echo "" - } >>"$DOCS_FILE" - - for keyword_file in "$PER_LANG_DIR"/keywords_*.txt; do - [ ! -f "$keyword_file" ] && continue - [ ! -s "$keyword_file" ] && continue - - # Extract language name from filename - lang=$(basename "$keyword_file" | sed 's/keywords_//; s/\.txt//') - doc_lang=$(lang_to_doc "$lang") - - # Format language name for display - case "$lang" in - c_cpp) display_lang="C/C++" ;; - javascript) display_lang="JavaScript" ;; - typescript) display_lang="TypeScript" ;; - python) display_lang="Python" ;; - rust) display_lang="Rust" ;; - go) display_lang="Go" ;; - ruby) display_lang="Ruby" ;; - java) display_lang="Java" ;; - shell) display_lang="Shell/Bash" ;; - *) display_lang="$lang" ;; - esac - - { - echo "### $display_lang Keywords" - echo "" - echo "| Keyword | Count | Documentation |" - echo "|---------|-------|---------------|" - } >>"$DOCS_FILE" - - head -n "$TOP_N" "$keyword_file" | while read -r count term; do - [ -z "$term" ] && continue - [[ $term =~ ^[#] ]] && continue # Skip comment lines - url=$(get_doc_url "$term" "$doc_lang") - echo "| \`$term\` | $count | [docs]($url) |" >>"$DOCS_FILE" - done - echo "" >>"$DOCS_FILE" - done - - # Process functions by language - { - echo "## Function/Method Calls" - echo "" - } >>"$DOCS_FILE" - - for func_file in "$PER_LANG_DIR"/functions_*.txt; do - [ ! -f "$func_file" ] && continue - [ ! -s "$func_file" ] && continue - - lang=$(basename "$func_file" | sed 's/functions_//; s/\.txt//') - doc_lang=$(lang_to_doc "$lang") - - case "$lang" in - c_cpp) display_lang="C/C++" ;; - javascript) display_lang="JavaScript" ;; - typescript) display_lang="TypeScript" ;; - python) display_lang="Python" ;; - rust) display_lang="Rust" ;; - go) display_lang="Go" ;; - ruby) display_lang="Ruby" ;; - java) display_lang="Java" ;; - shell) display_lang="Shell/Bash" ;; - *) display_lang="$lang" ;; - esac - - { - echo "### $display_lang Functions" - echo "" - echo "| Function | Count | Documentation |" - echo "|----------|-------|---------------|" - } >>"$DOCS_FILE" - - head -n "$TOP_N" "$func_file" | while read -r count term; do - [ -z "$term" ] && continue - [[ $term =~ ^(if|for|while|switch|catch|elif)$ ]] && continue - url=$(get_doc_url "$term" "$doc_lang") - echo "| \`$term()\` | $count | [docs]($url) |" >>"$DOCS_FILE" - done - echo "" >>"$DOCS_FILE" - done - - # Process imports by language - { - echo "## Imports/Includes" - echo "" - } >>"$DOCS_FILE" - - for import_file in "$PER_LANG_DIR"/imports_*.txt; do - [ ! -f "$import_file" ] && continue - [ ! -s "$import_file" ] && continue - - lang=$(basename "$import_file" | sed 's/imports_//; s/\.txt//') - doc_lang=$(lang_to_doc "$lang") - - case "$lang" in - c_cpp) display_lang="C/C++ (#include)" ;; - javascript) display_lang="JavaScript (import/require)" ;; - typescript) display_lang="TypeScript (import)" ;; - python) display_lang="Python (import/from)" ;; - rust) display_lang="Rust (use)" ;; - go) display_lang="Go (import)" ;; - ruby) display_lang="Ruby (require)" ;; - java) display_lang="Java (import)" ;; - shell) display_lang="Shell (source)" ;; - *) display_lang="$lang" ;; - esac - - { - echo "### $display_lang" - echo "" - echo "| Import | Count | Documentation |" - echo "|--------|-------|---------------|" - } >>"$DOCS_FILE" - - head -20 "$import_file" | while read -r count import; do - [ -z "$import" ] && continue - # For offline lookup, pass the full import line for better context - url=$(get_doc_url "" "$doc_lang" "$import") - if [ -z "$url" ] || [[ $url == *"search.html"* ]]; then - # Fallback: extract module and try again - module=$(echo "$import" | sed -E 's/.*[<"]([^">]+)[">].*/\1/' | sed 's|.*/||' | sed 's/\..*$//') - url=$(get_doc_url "$module" "$doc_lang") - fi - import_escaped="${import//|/\\|}" - echo "| \`$import_escaped\` | $count | [docs]($url) |" >>"$DOCS_FILE" - done - echo "" >>"$DOCS_FILE" - done - -else - # Fallback to combined files (old behavior) - echo -e "${YELLOW}No per-language files found, using combined analysis${NC}" - - if [ -f "$RESULTS_DIR/grep_keywords.txt" ]; then - { - echo "## Language Keywords" - echo "" - echo "| Keyword | Count | Documentation |" - echo "|---------|-------|---------------|" - } >>"$DOCS_FILE" - - head -n "$TOP_N" "$RESULTS_DIR/grep_keywords.txt" | while read -r count term; do - [ -z "$term" ] && continue - url=$(get_doc_url "$term" "$PRIMARY_LANG") - echo "| \`$term\` | $count | [docs]($url) |" >>"$DOCS_FILE" - done - echo "" >>"$DOCS_FILE" - fi - - if [ -f "$RESULTS_DIR/grep_function_calls.txt" ]; then - { - echo "## Function/Method Calls" - echo "" - echo "| Function | Count | Documentation |" - echo "|----------|-------|---------------|" - } >>"$DOCS_FILE" - - head -n "$TOP_N" "$RESULTS_DIR/grep_function_calls.txt" | while read -r count term; do - [ -z "$term" ] && continue - [[ $term =~ ^(if|for|while|switch|catch)$ ]] && continue - url=$(get_doc_url "$term" "$PRIMARY_LANG") - echo "| \`$term()\` | $count | [docs]($url) |" >>"$DOCS_FILE" - done - echo "" >>"$DOCS_FILE" - fi - - if [ -f "$RESULTS_DIR/grep_imports.txt" ]; then - { - echo "## Imports/Includes" - echo "" - echo "| Import | Count | Documentation |" - echo "|--------|-------|---------------|" - } >>"$DOCS_FILE" - - head -20 "$RESULTS_DIR/grep_imports.txt" | while read -r count import; do - [ -z "$import" ] && continue - module=$(echo "$import" | sed -E 's/.*[<"]([^">]+)[">].*/\1/' | sed 's|.*/||' | sed 's/\..*$//') - url=$(get_doc_url "$module" "$PRIMARY_LANG") - import_escaped="${import//|/\\|}" - echo "| \`$import_escaped\` | $count | [docs]($url) |" >>"$DOCS_FILE" - done - echo "" >>"$DOCS_FILE" - fi -fi - -{ - echo "" - echo "---" - echo "*Generated by analyze_repo.sh + generate_study_materials.sh*" -} >>"$DOCS_FILE" - -echo -e "${GREEN}Created: $DOCS_FILE${NC}" -#============================================================================== -# Generate Anki Cards (Tab-separated for import) -#============================================================================== -echo -e "${YELLOW}Generating Anki cards...${NC}" - -cat >"$ANKI_FILE" <<'EOF' -# Anki Import File -# Format: FrontBackTags -# Import with: File -> Import, select "Fields separated by: Tab" -# -# Card Types: -# 1. "What does X do?" - For functions/methods -# 2. "When to use X?" - For keywords/patterns -# 3. "What is the syntax for X?" - For language constructs -# -EOF - -# Generate cards for top keywords -if [ -f "$RESULTS_DIR/grep_keywords.txt" ]; then - echo "# Keywords" >>"$ANKI_FILE" - head -n "$TOP_N" "$RESULTS_DIR/grep_keywords.txt" | while read -r count term; do - [ -z "$term" ] && continue - url=$(get_doc_url "$term" "$PRIMARY_LANG") - - # Create different card types based on term type - case "$term" in - if | else | elif | elseif | switch | case | match) - echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tConditional control flow - executes code based on boolean conditions. See: $url\t${PRIMARY_LANG}::keywords::control-flow" >>"$ANKI_FILE" - ;; - for | while | loop | do | until) - echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tLoop construct - repeats code execution. See: $url\t${PRIMARY_LANG}::keywords::loops" >>"$ANKI_FILE" - ;; - try | except | catch | finally | raise | throw) - echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tException handling - manages errors and exceptional conditions. See: $url\t${PRIMARY_LANG}::keywords::exceptions" >>"$ANKI_FILE" - ;; - class | struct | interface | trait | impl) - echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tType definition - defines custom data structures. See: $url\t${PRIMARY_LANG}::keywords::types" >>"$ANKI_FILE" - ;; - def | fn | func | function) - echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tFunction definition - declares a reusable block of code. See: $url\t${PRIMARY_LANG}::keywords::functions" >>"$ANKI_FILE" - ;; - import | from | use | require | include) - echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tModule import - brings external code into current scope. See: $url\t${PRIMARY_LANG}::keywords::modules" >>"$ANKI_FILE" - ;; - async | await | yield) - echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tAsynchronous programming - handles concurrent operations. See: $url\t${PRIMARY_LANG}::keywords::async" >>"$ANKI_FILE" - ;; - *) - echo -e "What does the keyword \`$term\` do in $PRIMARY_LANG?\t[FILL: Look up at $url]\t${PRIMARY_LANG}::keywords" >>"$ANKI_FILE" - ;; - esac - done -fi - -# Generate cards for top functions -if [ -f "$RESULTS_DIR/grep_function_calls.txt" ]; then - { - echo "" - echo "# Functions" - } >>"$ANKI_FILE" - head -n "$TOP_N" "$RESULTS_DIR/grep_function_calls.txt" | while read -r count term; do - [ -z "$term" ] && continue - [[ $term =~ ^(if|for|while|switch|catch)$ ]] && continue - url=$(get_doc_url "$term" "$PRIMARY_LANG") - - echo -e "What does \`$term()\` do in $PRIMARY_LANG? (Used $count times)\t[FILL: Look up at $url]\t${PRIMARY_LANG}::functions" >>"$ANKI_FILE" - done -fi - -echo -e "${GREEN}Created: $ANKI_FILE${NC}" - -#============================================================================== -# Generate LLM Prompt for Anki Card Generation -#============================================================================== -echo -e "${YELLOW}Generating LLM prompt...${NC}" - -# Helper function to get doc link for a term -get_llm_doc_link() { - local term="$1" - local lang="$2" - local is_import="$3" # "true" if it's an import line - - # Check if it's an internal/project-specific item - if [[ $term =~ ^@/ ]] || [[ $term =~ ^\./ ]] || [[ $term =~ ^app\. ]] || [[ $term =~ ^src/ ]] || [[ $term =~ from\ \'@/ ]] || [[ $term =~ from\ \'\./ ]]; then - echo "[INTERNAL - SKIP]" - return - fi - - # Try offline lookup - local offline_result - if [ "$is_import" = "true" ]; then - offline_result=$("$LOOKUP_SCRIPT" --import "$term" "$lang" 2>/dev/null | grep "^/" | head -1) - else - offline_result=$("$LOOKUP_SCRIPT" "$term" "$lang" 2>/dev/null | grep "^File:" | head -1 | sed 's/^File: //') - fi - - if [ -n "$offline_result" ]; then - echo "$offline_result" - else - echo "[NO OFFLINE DOC]" - fi -} - -# Generate keywords with doc links -generate_keywords_with_docs() { - local keywords_file="$RESULTS_DIR/grep_keywords.txt" - [ ! -f "$keywords_file" ] && echo "No keywords found" && return - - head -n "$TOP_N" "$keywords_file" | grep -v '^#' | while read -r line; do - local count - count=$(echo "$line" | awk '{print $1}') - local keyword - keyword=$(echo "$line" | awk '{print $2}') - [ -z "$keyword" ] && continue - local doc_link - doc_link=$(get_llm_doc_link "$keyword" "$PRIMARY_LANG" "false") - echo "$count $keyword → $doc_link" - done -} - -# Generate functions with doc links -generate_functions_with_docs() { - local functions_file="$RESULTS_DIR/grep_function_calls.txt" - [ ! -f "$functions_file" ] && echo "No functions found" && return - - head -n "$TOP_N" "$functions_file" | grep -v '^#' | while read -r line; do - local count - count=$(echo "$line" | awk '{print $1}') - local func - func=$(echo "$line" | awk '{print $2}') - - # Skip single-letter functions (minified code) or empty - if [ -z "$func" ] || [ ${#func} -le 1 ]; then - continue - fi - - local doc_link - doc_link=$(get_llm_doc_link "$func" "$PRIMARY_LANG" "false") - echo "$count $func() → $doc_link" - done -} - -# Generate imports with doc links -generate_imports_with_docs() { - local imports_file="$RESULTS_DIR/grep_imports.txt" - [ ! -f "$imports_file" ] && echo "No imports found" && return - - head -20 "$imports_file" | grep -v '^#' | while read -r line; do - local count - count=$(echo "$line" | awk '{print $1}') - local import_stmt - import_stmt=$(echo "$line" | cut -d' ' -f2-) - [ -z "$import_stmt" ] && continue - - # Check if internal import - if [[ $import_stmt =~ @/ ]] || [[ $import_stmt =~ \./ ]] || [[ $import_stmt =~ from\ app\. ]] || [[ $import_stmt =~ from\ src\. ]]; then - echo "$count $import_stmt → [INTERNAL - SKIP]" - else - local doc_link - doc_link=$(get_llm_doc_link "$import_stmt" "$PRIMARY_LANG" "true") - echo "$count $import_stmt → $doc_link" - fi - done -} - -cat >"$LLM_PROMPT_FILE" <<'PROMPT_HEADER' -# LLM Prompt: Generate Anki Flashcards - -You are creating Anki flashcards from code analysis. - -## CRITICAL INSTRUCTIONS - -1. **READ DOCS VIA TERMINAL** - Use the `cat` command to read each .md file: - ``` - cat /home/kuhy/.local/share/offline-docs/mdn-content/files/en-us/web/javascript/reference/statements/const/index.md - ``` -2. **DO NOT USE YOUR OWN KNOWLEDGE** - Base flashcards ONLY on the content you read from the files -3. **IF YOU CANNOT READ A FILE** - Report: "ERROR: Cannot read [path]" and skip that item -4. **NEVER FALL BACK TO GENERAL KNOWLEDGE** - If you can't read the file, skip it entirely -5. **READ ONE FILE AT A TIME** - Run cat for each topic before creating its flashcards - -PROMPT_HEADER - -cat >>"$LLM_PROMPT_FILE" <>"$LLM_PROMPT_FILE" <<'PROMPT_FOOTER' - -## Guidelines - -**CRITICAL - Keep answers EXTREMELY short:** -- Most answers should be **1-2 words** or **1 sentence** -- It's common and expected for an answer to be just: "Returns an array" or "Immutable" -- 2 sentences = longer answer, 3 sentences = absolute maximum (rare) -- Each flashcard tests ONE atomic piece of knowledge - -**NO DUPLICATES:** -- Before creating a card, check if you already created a similar question -- Each unique fact should appear in EXACTLY ONE card -- Do NOT create multiple cards asking the same thing with slightly different wording - -**What to include:** -- Concept cards: "What is X?" / "What does X do?" -- Syntax cards: "How do you write X?" (brief code snippet) -- Comparison cards: "X vs Y - what's the difference?" - -**What to SKIP (do NOT create cards for):** -- MDN frontmatter fields: title, slug, page-type, browser-compat, spec-urls -- YAML metadata between `---` markers at the start of files -- Any line that looks like metadata (key: value at start of doc) -- Empty answers - if you can't find content for the back, skip the card entirely - -**FINAL CARD FOR EACH TOPIC (EXCEPTION TO SHORT ANSWER RULE):** -- Add EXACTLY ONE full documentation card per topic (no duplicates!) -- Question: `[Topic] - Full MDN Documentation` -- Answer: Copy the .md file content STARTING AFTER the `---` frontmatter block -- Skip the YAML frontmatter (everything between the first two `---` lines) -- Do NOT create this card twice for the same topic - -**Skipped items - please review:** -- Items marked `[INTERNAL - SKIP]` are project-specific utilities - I skipped them -- Items marked `[NO OFFLINE DOC]` are third-party libraries without bundled docs -- If you want flashcards for skipped items, tell me which ones to include - -## OUTPUT: CREATE AN ANKI FILE - -**CREATE A FILE DIRECTLY** - Do not just output text. Use your file creation tool to create: - -**File path:** `~/.local/share/study-materials/anki_generated.txt` - -**Format:** Tab-separated values (TSV) with Anki metadata headers: - -``` -#separator:tab -#deck:CodeStudy::JavaScript -#notetype:CodeCard -#columns:Front Back Tags -What does const declare?Block-scoped variables with immutable bindings.javascript declarations -``` - -**Required headers at top of file:** -- `#separator:tab` - Specifies tab as delimiter -- `#deck:CodeStudy::[Language]` - Creates deck "CodeStudy" with sub-deck for language (e.g., CodeStudy::JavaScript) -- `#notetype:CodeCard` - Uses custom note type "CodeCard" (Anki will create if doesn't exist) -- `#columns:Front Back Tags` - Column headers (tab-separated) - -**Rules:** -- Use ACTUAL `` tags (not escaped <code>) -- Use `
` for line breaks within fields -- Use `
` for code blocks
-- Tags are space-separated
-- Escape any literal tabs within content as spaces
-
-**Example file content:**
-```
-#separator:tab
-#deck:CodeStudy::JavaScript
-#notetype:CodeCard
-#columns:Front	Back	Tags
-What does const declare?Block-scoped variables with immutable bindings.javascript declarations
-Can const be reassigned?No, throws TypeError.javascript declarations
-const - Full Documentation
[ENTIRE CONTENT OF const/index.md FILE]
javascript declarations full-doc -``` - -**After creating the file**, tell the user: -- File created at: ~/.local/share/study-materials/anki_generated.txt -- Import in Anki: File → Import → select the file -- Deck: CodeStudy::[Language], Note type: CodeCard ---- - -**Important:** -- Process only 5-10 items at a time to maintain quality -- Focus on items with offline documentation paths -- Output ONLY the TSV lines, no extra formatting or markdown -PROMPT_FOOTER - -echo -e "${GREEN}Created: $LLM_PROMPT_FILE${NC}" +# The three generation phases, in the order their outputs reference each +# other. Defined above and called here so the file can be split into libs. +generate_doc_links +generate_anki_cards +generate_llm_prompt #============================================================================== # Summary diff --git a/linux_configuration/scripts/single_use/utils/lib/study_doc_lookup.sh b/linux_configuration/scripts/single_use/utils/lib/study_doc_lookup.sh new file mode 100755 index 00000000..4a1263be --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/study_doc_lookup.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# lib/study_doc_lookup.sh — resolve a term to documentation. +# +# get_doc_url dispatches to the per-language builders in study_doc_urls*.sh; +# lookup_offline prefers a local mirror when one is installed, so a generated +# card can cite a file an assistant can actually read rather than a URL it +# would have to fetch. detect_language picks the primary language out of the +# tokei stats the analysis step wrote. + +#============================================================================== +# Offline Documentation Lookup (preferred if available) +#============================================================================== +lookup_offline() { + local term="$1" + local lang="$2" + # Optional: the full import line, for import-aware lookups. Documented as + # optional and genuinely called without it, so it is defaulted rather than + # required -- under `set -u` a bare "$3" aborts instead. + local import_line="${3:-}" + + if ! $USE_OFFLINE_DOCS; then + return 1 + fi + + local result + if [ -n "$import_line" ]; then + # Use import-aware lookup - get the line with the file path + result=$("$LOOKUP_SCRIPT" --import "$import_line" "$lang" 2>/dev/null | grep "^/" | head -1) + else + result=$("$LOOKUP_SCRIPT" "$term" "$lang" 2>/dev/null | grep "^File:" | head -1 | sed 's/^File: //') + fi + + if [ -n "$result" ]; then + # Extract file path (before the | separator) + local file_path + file_path=$(echo "$result" | cut -d'|' -f1) + if [ -n "$file_path" ]; then + echo "$file_path" + return 0 + fi + fi + + return 1 +} + +#============================================================================== +# Get documentation URL for a term based on detected language +#============================================================================== +get_doc_url() { + local term="$1" + local lang="$2" + # Optional, as above: 8 of the 9 call sites pass only a term and a language. + local import_line="${3:-}" + + # Try offline docs first + local offline_result + offline_result=$(lookup_offline "$term" "$lang" "$import_line") + if [ -n "$offline_result" ]; then + echo "$offline_result" + return 0 + fi + + # For TypeScript, also try JavaScript offline docs (most TS keywords are JS) + if [[ $lang == "typescript" || $lang == "ts" || $lang == "tsx" ]]; then + offline_result=$(lookup_offline "$term" "js" "$import_line") + if [ -n "$offline_result" ]; then + echo "$offline_result" + return 0 + fi + fi + + # Fall back to online URLs + case "$lang" in + python | py) + python_doc_url "$term" + ;; + javascript | js | jsx) + js_doc_url "$term" + ;; + typescript | ts | tsx) + # For TypeScript, try JS doc first (since most keywords are shared) + # Only use TS-specific docs for TS-only features + case "$term" in + interface | type | enum | namespace | declare | readonly | abstract | implements | keyof | infer | as | is | asserts | satisfies | override | Partial | Required | Readonly | Record | Pick | Omit | Exclude | Extract | NonNullable | ReturnType | Parameters | InstanceType | Awaited) + ts_doc_url "$term" + ;; + *) + js_doc_url "$term" + ;; + esac + ;; + c) + c_doc_url "$term" + ;; + cpp | c++ | cc | cxx) + cpp_doc_url "$term" + ;; + rust | rs) + rust_doc_url "$term" + ;; + go) + go_doc_url "$term" + ;; + ruby | rb) + ruby_doc_url "$term" + ;; + java) + java_doc_url "$term" + ;; + shell | bash | sh) + shell_doc_url "$term" + ;; + *) + echo "https://devdocs.io/#q=$term" + ;; + esac +} + +#============================================================================== +# Detect primary language from results +#============================================================================== +detect_language() { + if [ -f "$RESULTS_DIR/tokei_stats.txt" ]; then + # Parse tokei output to find most used language + grep -E "^\s+(Python|JavaScript|TypeScript|C\+\+|C |Rust|Go|Ruby|Java|Shell)" "$RESULTS_DIR/tokei_stats.txt" 2>/dev/null | + head -1 | + awk '{print tolower($1)}' | + sed 's/c++/cpp/' + else + echo "unknown" + fi +} diff --git a/linux_configuration/scripts/single_use/utils/lib/study_doc_urls.sh b/linux_configuration/scripts/single_use/utils/lib/study_doc_urls.sh new file mode 100755 index 00000000..b1c124c8 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/study_doc_urls.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# lib/study_doc_urls.sh — per-language documentation URL builders for the +# C-family, JS/TS and Python. +# +# Sourced by generate_study_materials.sh. Each takes a term and returns the +# canonical online doc URL for it, or an empty string when the language has no +# stable URL shape for that term. + +# Python documentation +python_doc_url() { + local term="$1" + # Category (keyword, builtin, module), reserved for future use and never + # read. Defaulted rather than required: get_doc_url has always called this + # with one argument, which only worked because the entry script runs under + # `set -e` without `-u`, so "$2" silently expanded to empty. Any caller that + # does enable `set -u` -- a test, or a future lib that sources this -- would + # otherwise abort here on an argument nothing uses. + local _type="${2:-}" + + case "$term" in + # Keywords + if | else | elif | for | while | try | except | finally | with | as | import | from | def | class | return | yield | raise | pass | break | continue | and | or | not | in | is | lambda | global | nonlocal | assert | del | True | False | None | async | await) + echo "https://docs.python.org/3/reference/compound_stmts.html" + ;; + # Built-in functions + print | len | range | type | str | int | float | list | dict | set | tuple | bool | open | input | format | sorted | reversed | enumerate | zip | map | filter | any | all | sum | min | max | abs | round | isinstance | issubclass | hasattr | getattr | setattr | delattr | callable | iter | next | super | property | staticmethod | classmethod | vars | dir | help | id | hash | repr | ascii | bin | hex | oct | chr | ord | eval | exec | compile) + echo "https://docs.python.org/3/library/functions.html#$term" + ;; + # Common modules + os | sys | re | json | datetime | collections | itertools | functools | pathlib | subprocess | threading | multiprocessing | asyncio | typing | dataclasses | unittest | pytest | logging | argparse | configparser) + echo "https://docs.python.org/3/library/$term.html" + ;; + # Testing + MagicMock | Mock | patch | PropertyMock) + echo "https://docs.python.org/3/library/unittest.mock.html" + ;; + *) + echo "https://docs.python.org/3/search.html?q=$term" + ;; + esac +} + +# JavaScript/TypeScript documentation (MDN) +js_doc_url() { + local term="$1" + + case "$term" in + # Keywords & statements + if | else | for | while | do | switch | case | break | continue | return | throw | try | catch | finally | function | class | const | let | var | new | this | super | import | export | default | async | await | yield | typeof | instanceof | in | of | delete | void) + echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements" + ;; + # Global objects + Array | Object | String | Number | Boolean | Symbol | Map | Set | WeakMap | WeakSet | Date | RegExp | Error | Promise | Proxy | Reflect | JSON | Math | Intl) + echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/$term" + ;; + # Array methods + map | filter | reduce | forEach | find | findIndex | some | every | includes | indexOf | slice | splice | concat | join | push | pop | shift | unshift | sort | reverse | flat | flatMap) + echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/$term" + ;; + # String methods + split | replace | match | search | substring | substr | toLowerCase | toUpperCase | trim | padStart | padEnd | startsWith | endsWith | charAt | charCodeAt) + echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/$term" + ;; + # Promise methods + then | resolve | reject | all | race | allSettled | any) + echo "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/$term" + ;; + # Common Web APIs + fetch | console | document | window | localStorage | sessionStorage | setTimeout | setInterval | addEventListener | querySelector | querySelectorAll) + echo "https://developer.mozilla.org/en-US/docs/Web/API" + ;; + *) + echo "https://developer.mozilla.org/en-US/search?q=$term" + ;; + esac +} + +# TypeScript-specific documentation +ts_doc_url() { + local term="$1" + + case "$term" in + interface | type | enum | namespace | declare | readonly | abstract | implements | extends | keyof | typeof | infer | as | is | asserts | satisfies | override) + echo "https://www.typescriptlang.org/docs/handbook/2/everyday-types.html" + ;; + Partial | Required | Readonly | Record | Pick | Omit | Exclude | Extract | NonNullable | ReturnType | Parameters | InstanceType | Awaited) + echo "https://www.typescriptlang.org/docs/handbook/utility-types.html" + ;; + *) + # Fall back to JS docs for runtime features + js_doc_url "$term" + ;; + esac +} + +# C documentation +c_doc_url() { + local term="$1" + + case "$term" in + # Keywords + if | else | for | while | do | switch | case | break | continue | return | goto | sizeof | typedef | struct | union | enum | const | static | extern | register | volatile | inline | restrict | _Bool | _Complex | _Imaginary | _Alignas | _Alignof | _Atomic | _Generic | _Noreturn | _Static_assert | _Thread_local) + echo "https://en.cppreference.com/w/c/keyword/$term" + ;; + # Standard library headers + stdio | stdlib | string | math | time | ctype | stdint | stdbool | stddef | limits | float | errno | assert | signal | setjmp | stdarg | locale) + echo "https://en.cppreference.com/w/c/header/${term}.h" + ;; + # Common functions + printf | fprintf | sprintf | snprintf | scanf | fscanf | sscanf | fopen | fclose | fread | fwrite | fgets | fputs | fseek | ftell | rewind | fflush) + echo "https://en.cppreference.com/w/c/io" + ;; + malloc | calloc | realloc | free | memcpy | memmove | memset | memcmp) + echo "https://en.cppreference.com/w/c/memory" + ;; + strlen | strcpy | strncpy | strcat | strncat | strcmp | strncmp | strchr | strrchr | strstr | strtok) + echo "https://en.cppreference.com/w/c/string/byte" + ;; + *) + echo "https://en.cppreference.com/mwiki/index.php?search=$term" + ;; + esac +} + +# C++ documentation +cpp_doc_url() { + local term="$1" + + case "$term" in + # C++ specific keywords + class | public | private | protected | virtual | override | final | explicit | mutable | constexpr | consteval | constinit | concept | requires | co_await | co_yield | co_return | nullptr | noexcept | decltype | auto | template | typename | namespace | using | new | delete | throw | try | catch | static_cast | dynamic_cast | const_cast | reinterpret_cast) + echo "https://en.cppreference.com/w/cpp/keyword/$term" + ;; + # STL containers + vector | list | deque | array | forward_list | set | map | unordered_set | unordered_map | multiset | multimap | stack | queue | priority_queue) + echo "https://en.cppreference.com/w/cpp/container/$term" + ;; + # STL algorithms + sort | find | copy | move | transform | accumulate | count | remove | unique | reverse | rotate | shuffle | partition | merge | binary_search | lower_bound | upper_bound) + echo "https://en.cppreference.com/w/cpp/algorithm/$term" + ;; + # Smart pointers + unique_ptr | shared_ptr | weak_ptr | make_unique | make_shared) + echo "https://en.cppreference.com/w/cpp/memory/$term" + ;; + # Common classes + string | string_view | optional | variant | any | tuple | pair | function | bind | thread | mutex | future | promise | chrono) + echo "https://en.cppreference.com/w/cpp/utility" + ;; + *) + # Try C docs as fallback + c_doc_url "$term" + ;; + esac +} diff --git a/linux_configuration/scripts/single_use/utils/lib/study_doc_urls_more.sh b/linux_configuration/scripts/single_use/utils/lib/study_doc_urls_more.sh new file mode 100755 index 00000000..6fd302f5 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/study_doc_urls_more.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# lib/study_doc_urls_more.sh — documentation URL builders for Rust, Go, Ruby, +# Java and shell. +# +# Split from study_doc_urls.sh purely to keep both files under the 250-line +# cap; the two are one subject and are always sourced together. + +# Rust documentation +rust_doc_url() { + local term="$1" + + case "$term" in + # Keywords + fn | let | mut | const | static | if | else | match | loop | while | for | in | break | continue | return | struct | enum | impl | trait | type | where | pub | mod | use | crate | self | super | async | await | move | ref | dyn | unsafe | extern) + echo "https://doc.rust-lang.org/std/keyword.$term.html" + ;; + # Common types + Option | Result | Vec | String | Box | Rc | Arc | Cell | RefCell | Mutex | RwLock | HashMap | HashSet | BTreeMap | BTreeSet) + echo "https://doc.rust-lang.org/std/$term" + ;; + # Traits + Clone | Copy | Debug | Default | Eq | PartialEq | Ord | PartialOrd | Hash | Display | From | Into | AsRef | AsMut | Deref | DerefMut | Iterator | IntoIterator | Send | Sync) + echo "https://doc.rust-lang.org/std/$term" + ;; + # Macros + println | print | format | vec | panic | assert | assert_eq | assert_ne | debug_assert | todo | unimplemented | unreachable) + echo "https://doc.rust-lang.org/std/macro.$term.html" + ;; + *) + echo "https://doc.rust-lang.org/std/?search=$term" + ;; + esac +} + +# Go documentation +go_doc_url() { + local term="$1" + + case "$term" in + # Keywords + func | var | const | type | struct | interface | map | chan | go | select | defer | if | else | for | range | switch | case | default | break | continue | return | goto | fallthrough | package | import) + echo "https://go.dev/ref/spec" + ;; + # Built-in functions + make | new | len | cap | append | copy | delete | close | panic | recover | print | println | complex | real | imag) + echo "https://pkg.go.dev/builtin#$term" + ;; + # Common packages + fmt | os | io | net | http | json | time | strings | strconv | errors | context | sync | testing | reflect | regexp | sort | math | crypto | encoding | bufio | bytes | path | filepath) + echo "https://pkg.go.dev/$term" + ;; + *) + echo "https://pkg.go.dev/search?q=$term" + ;; + esac +} + +# Ruby documentation +ruby_doc_url() { + local term="$1" + + case "$term" in + # Keywords + if | else | elsif | unless | case | when | while | until | for | do | end | begin | rescue | ensure | raise | return | break | next | redo | retry | yield | def | class | module | self | super | nil | true | false | and | or | not | in | then | alias | defined | __FILE__ | __LINE__ | __ENCODING__) + echo "https://ruby-doc.org/docs/keywords/1.9/" + ;; + # Core classes + String | Array | Hash | Integer | Float | Symbol | Range | Regexp | Time | Date | File | Dir | IO | Proc | Lambda | Method | Thread | Mutex | Fiber) + echo "https://ruby-doc.org/core/classes/$term.html" + ;; + # Enumerable methods + each | map | select | reject | find | reduce | inject | collect | detect | sort | sort_by | group_by | partition | any | all | none | one | count | first | last | take | drop) + echo "https://ruby-doc.org/core/Enumerable.html" + ;; + *) + echo "https://ruby-doc.org/search.html?q=$term" + ;; + esac +} + +# Java documentation +java_doc_url() { + local term="$1" + + case "$term" in + # Keywords + if | else | for | while | do | switch | case | break | continue | return | throw | try | catch | finally | class | interface | enum | extends | implements | new | this | super | static | final | abstract | public | private | protected | void | null | true | false | instanceof | synchronized | volatile | transient | native | strictfp | assert | default | package | import) + echo "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/" + ;; + # Common classes + String | Integer | Long | Double | Float | Boolean | Character | Object | Class | System | Math | Arrays | Collections | List | ArrayList | LinkedList | Map | HashMap | TreeMap | Set | HashSet | TreeSet | Queue | Stack | Optional | Stream) + echo "https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/$term.html" + ;; + *) + echo "https://docs.oracle.com/en/java/javase/17/docs/api/search.html?q=$term" + ;; + esac +} + +# Shell documentation +shell_doc_url() { + local term="$1" + + case "$term" in + # Built-in commands + if | then | else | elif | fi | for | while | until | do | done | case | 'esac' | in | function | select | time | coproc) + echo "https://www.gnu.org/software/bash/manual/bash.html#Conditional-Constructs" + ;; + echo | printf | read | declare | local | export | unset | set | shopt | alias | source | eval | exec | exit | return | break | continue | shift | trap | wait | kill | jobs | bg | fg | disown | suspend | logout | cd | pwd | pushd | popd | dirs | type | which | command | builtin | enable | help | hash | bind | complete | compgen | compopt) + echo "https://www.gnu.org/software/bash/manual/bash.html#Shell-Builtin-Commands" + ;; + # Common external commands + grep | sed | awk | find | xargs | sort | uniq | cut | tr | head | tail | wc | cat | tee | diff | patch | tar | gzip | zip | curl | wget | ssh | scp | rsync | git | make | chmod | chown | chgrp | ln | cp | mv | rm | mkdir | rmdir | touch | ls | stat | file | df | du | free | top | ps | pkill | pgrep | nohup | screen | tmux) + echo "https://man7.org/linux/man-pages/man1/$term.1.html" + ;; + *) + echo "https://www.gnu.org/software/bash/manual/bash.html" + ;; + esac +} diff --git a/linux_configuration/scripts/single_use/utils/lib/study_gen_anki.sh b/linux_configuration/scripts/single_use/utils/lib/study_gen_anki.sh new file mode 100755 index 00000000..f89fa1f7 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/study_gen_anki.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# lib/study_gen_anki.sh — write anki_cards.txt, the tab-separated import file: +# one card per top keyword and one per top function call. + +# Write the tab-separated Anki import file: one card per top keyword and +# one per top function call. +generate_anki_cards() { + #============================================================================== + # Generate Anki Cards (Tab-separated for import) + #============================================================================== + echo -e "${YELLOW}Generating Anki cards...${NC}" + + cat >"$ANKI_FILE" <<'EOF' +# Anki Import File +# Format: FrontBackTags +# Import with: File -> Import, select "Fields separated by: Tab" +# +# Card Types: +# 1. "What does X do?" - For functions/methods +# 2. "When to use X?" - For keywords/patterns +# 3. "What is the syntax for X?" - For language constructs +# +EOF + + # Generate cards for top keywords + if [ -f "$RESULTS_DIR/grep_keywords.txt" ]; then + echo "# Keywords" >>"$ANKI_FILE" + { grep -v '^#' "$RESULTS_DIR/grep_keywords.txt" || true; } | head -n "$TOP_N" | while read -r count term; do + [ -z "$term" ] && continue + url=$(get_doc_url "$term" "$PRIMARY_LANG") + + # Create different card types based on term type + case "$term" in + if | else | elif | elseif | switch | case | match) + echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tConditional control flow - executes code based on boolean conditions. See: $url\t${PRIMARY_LANG}::keywords::control-flow" >>"$ANKI_FILE" + ;; + for | while | loop | do | until) + echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tLoop construct - repeats code execution. See: $url\t${PRIMARY_LANG}::keywords::loops" >>"$ANKI_FILE" + ;; + try | except | catch | finally | raise | throw) + echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tException handling - manages errors and exceptional conditions. See: $url\t${PRIMARY_LANG}::keywords::exceptions" >>"$ANKI_FILE" + ;; + class | struct | interface | trait | impl) + echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tType definition - defines custom data structures. See: $url\t${PRIMARY_LANG}::keywords::types" >>"$ANKI_FILE" + ;; + def | fn | func | function) + echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tFunction definition - declares a reusable block of code. See: $url\t${PRIMARY_LANG}::keywords::functions" >>"$ANKI_FILE" + ;; + import | from | use | require | include) + echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tModule import - brings external code into current scope. See: $url\t${PRIMARY_LANG}::keywords::modules" >>"$ANKI_FILE" + ;; + async | await | yield) + echo -e "What is the purpose of \`$term\` in $PRIMARY_LANG?\tAsynchronous programming - handles concurrent operations. See: $url\t${PRIMARY_LANG}::keywords::async" >>"$ANKI_FILE" + ;; + *) + echo -e "What does the keyword \`$term\` do in $PRIMARY_LANG?\t[FILL: Look up at $url]\t${PRIMARY_LANG}::keywords" >>"$ANKI_FILE" + ;; + esac + done + fi + + # Generate cards for top functions + if [ -f "$RESULTS_DIR/grep_function_calls.txt" ]; then + { + echo "" + echo "# Functions" + } >>"$ANKI_FILE" + { grep -v '^#' "$RESULTS_DIR/grep_function_calls.txt" || true; } | head -n "$TOP_N" | while read -r count term; do + [ -z "$term" ] && continue + [[ $term =~ ^(if|for|while|switch|catch)$ ]] && continue + url=$(get_doc_url "$term" "$PRIMARY_LANG") + + echo -e "What does \`$term()\` do in $PRIMARY_LANG? (Used $count times)\t[FILL: Look up at $url]\t${PRIMARY_LANG}::functions" >>"$ANKI_FILE" + done + fi + + echo -e "${GREEN}Created: $ANKI_FILE${NC}" + +} diff --git a/linux_configuration/scripts/single_use/utils/lib/study_gen_docs.sh b/linux_configuration/scripts/single_use/utils/lib/study_gen_docs.sh new file mode 100755 index 00000000..14dacc25 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/study_gen_docs.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# lib/study_gen_docs.sh — write documentation_links.md, the markdown index of +# every top keyword, function call and import the analysis found. + +# Write the markdown documentation-link file covering every top keyword, +# function call and import in the analysis. +generate_doc_links() { + #============================================================================== + # Generate Documentation Links (Markdown) + #============================================================================== + echo -e "${YELLOW}Generating documentation links...${NC}" + + cat >"$DOCS_FILE" <<'EOF' +# Documentation Links for Code Review + +This document contains links to official documentation for the most commonly used +functions, keywords, and patterns found in the analyzed codebase. + +**Note:** Items are grouped by language for accurate documentation links. + +--- + +EOF + + # Check for per-language files + PER_LANG_DIR="$RESULTS_DIR/per_language" + + if [ -d "$PER_LANG_DIR" ]; then + echo -e "${GREEN}Using per-language analysis files${NC}" + + # Map internal lang names to doc function names + lang_to_doc() { + case "$1" in + c_cpp) echo "cpp" ;; + javascript) echo "js" ;; + typescript) echo "ts" ;; + shell) echo "bash" ;; + *) echo "$1" ;; + esac + } + + # Process keywords by language + { + echo "## Language Keywords" + echo "" + } >>"$DOCS_FILE" + + for keyword_file in "$PER_LANG_DIR"/keywords_*.txt; do + [ ! -f "$keyword_file" ] && continue + [ ! -s "$keyword_file" ] && continue + + # Extract language name from filename + lang=$(basename "$keyword_file" | sed 's/keywords_//; s/\.txt//') + doc_lang=$(lang_to_doc "$lang") + + # Format language name for display + case "$lang" in + c_cpp) display_lang="C/C++" ;; + javascript) display_lang="JavaScript" ;; + typescript) display_lang="TypeScript" ;; + python) display_lang="Python" ;; + rust) display_lang="Rust" ;; + go) display_lang="Go" ;; + ruby) display_lang="Ruby" ;; + java) display_lang="Java" ;; + shell) display_lang="Shell/Bash" ;; + *) display_lang="$lang" ;; + esac + + { + echo "### $display_lang Keywords" + echo "" + echo "| Keyword | Count | Documentation |" + echo "|---------|-------|---------------|" + } >>"$DOCS_FILE" + + { grep -v '^#' "$keyword_file" || true; } | head -n "$TOP_N" | while read -r count term; do + [ -z "$term" ] && continue + url=$(get_doc_url "$term" "$doc_lang") + echo "| \`$term\` | $count | [docs]($url) |" >>"$DOCS_FILE" + done + echo "" >>"$DOCS_FILE" + done + + # Process functions by language + { + echo "## Function/Method Calls" + echo "" + } >>"$DOCS_FILE" + + for func_file in "$PER_LANG_DIR"/functions_*.txt; do + [ ! -f "$func_file" ] && continue + [ ! -s "$func_file" ] && continue + + lang=$(basename "$func_file" | sed 's/functions_//; s/\.txt//') + doc_lang=$(lang_to_doc "$lang") + + case "$lang" in + c_cpp) display_lang="C/C++" ;; + javascript) display_lang="JavaScript" ;; + typescript) display_lang="TypeScript" ;; + python) display_lang="Python" ;; + rust) display_lang="Rust" ;; + go) display_lang="Go" ;; + ruby) display_lang="Ruby" ;; + java) display_lang="Java" ;; + shell) display_lang="Shell/Bash" ;; + *) display_lang="$lang" ;; + esac + + { + echo "### $display_lang Functions" + echo "" + echo "| Function | Count | Documentation |" + echo "|----------|-------|---------------|" + } >>"$DOCS_FILE" + + { grep -v '^#' "$func_file" || true; } | head -n "$TOP_N" | while read -r count term; do + [ -z "$term" ] && continue + [[ $term =~ ^(if|for|while|switch|catch|elif)$ ]] && continue + url=$(get_doc_url "$term" "$doc_lang") + echo "| \`$term()\` | $count | [docs]($url) |" >>"$DOCS_FILE" + done + echo "" >>"$DOCS_FILE" + done + + # Process imports by language + { + echo "## Imports/Includes" + echo "" + } >>"$DOCS_FILE" + + for import_file in "$PER_LANG_DIR"/imports_*.txt; do + [ ! -f "$import_file" ] && continue + [ ! -s "$import_file" ] && continue + + lang=$(basename "$import_file" | sed 's/imports_//; s/\.txt//') + doc_lang=$(lang_to_doc "$lang") + + case "$lang" in + c_cpp) display_lang="C/C++ (#include)" ;; + javascript) display_lang="JavaScript (import/require)" ;; + typescript) display_lang="TypeScript (import)" ;; + python) display_lang="Python (import/from)" ;; + rust) display_lang="Rust (use)" ;; + go) display_lang="Go (import)" ;; + ruby) display_lang="Ruby (require)" ;; + java) display_lang="Java (import)" ;; + shell) display_lang="Shell (source)" ;; + *) display_lang="$lang" ;; + esac + + { + echo "### $display_lang" + echo "" + echo "| Import | Count | Documentation |" + echo "|--------|-------|---------------|" + } >>"$DOCS_FILE" + + { grep -v '^#' "$import_file" || true; } | head -n "$TOP_N" | while read -r count import; do + [ -z "$import" ] && continue + # For offline lookup, pass the full import line for better context + url=$(get_doc_url "" "$doc_lang" "$import") + if [ -z "$url" ] || [[ $url == *"search.html"* ]]; then + # Fallback: extract module and try again + module=$(echo "$import" | sed -E 's/.*[<"]([^">]+)[">].*/\1/' | sed 's|.*/||' | sed 's/\..*$//') + url=$(get_doc_url "$module" "$doc_lang") + fi + import_escaped="${import//|/\\|}" + echo "| \`$import_escaped\` | $count | [docs]($url) |" >>"$DOCS_FILE" + done + echo "" >>"$DOCS_FILE" + done + + else + # Fallback to combined files (old behavior) + echo -e "${YELLOW}No per-language files found, using combined analysis${NC}" + + if [ -f "$RESULTS_DIR/grep_keywords.txt" ]; then + { + echo "## Language Keywords" + echo "" + echo "| Keyword | Count | Documentation |" + echo "|---------|-------|---------------|" + } >>"$DOCS_FILE" + + { grep -v '^#' "$RESULTS_DIR/grep_keywords.txt" || true; } | head -n "$TOP_N" | while read -r count term; do + [ -z "$term" ] && continue + url=$(get_doc_url "$term" "$PRIMARY_LANG") + echo "| \`$term\` | $count | [docs]($url) |" >>"$DOCS_FILE" + done + echo "" >>"$DOCS_FILE" + fi + + if [ -f "$RESULTS_DIR/grep_function_calls.txt" ]; then + { + echo "## Function/Method Calls" + echo "" + echo "| Function | Count | Documentation |" + echo "|----------|-------|---------------|" + } >>"$DOCS_FILE" + + { grep -v '^#' "$RESULTS_DIR/grep_function_calls.txt" || true; } | head -n "$TOP_N" | while read -r count term; do + [ -z "$term" ] && continue + [[ $term =~ ^(if|for|while|switch|catch)$ ]] && continue + url=$(get_doc_url "$term" "$PRIMARY_LANG") + echo "| \`$term()\` | $count | [docs]($url) |" >>"$DOCS_FILE" + done + echo "" >>"$DOCS_FILE" + fi + + if [ -f "$RESULTS_DIR/grep_imports.txt" ]; then + { + echo "## Imports/Includes" + echo "" + echo "| Import | Count | Documentation |" + echo "|--------|-------|---------------|" + } >>"$DOCS_FILE" + + { grep -v '^#' "$RESULTS_DIR/grep_imports.txt" || true; } | head -n "$TOP_N" | while read -r count import; do + [ -z "$import" ] && continue + module=$(echo "$import" | sed -E 's/.*[<"]([^">]+)[">].*/\1/' | sed 's|.*/||' | sed 's/\..*$//') + url=$(get_doc_url "$module" "$PRIMARY_LANG") + import_escaped="${import//|/\\|}" + echo "| \`$import_escaped\` | $count | [docs]($url) |" >>"$DOCS_FILE" + done + echo "" >>"$DOCS_FILE" + fi + fi + + { + echo "" + echo "---" + echo "*Generated by analyze_repo.sh + generate_study_materials.sh*" + } >>"$DOCS_FILE" + + echo -e "${GREEN}Created: $DOCS_FILE${NC}" +} diff --git a/linux_configuration/scripts/single_use/utils/lib/study_gen_llm.sh b/linux_configuration/scripts/single_use/utils/lib/study_gen_llm.sh new file mode 100755 index 00000000..005954e5 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/study_gen_llm.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash +# lib/study_gen_llm.sh — write llm_anki_prompt.md. +# +# Its nested get_llm_doc_link and generate_*_with_docs helpers stay nested: +# they were defined inside this phase in the pre-split script too, and they +# close over PRIMARY_LANG and TOP_N from it. + +# Write the LLM prompt file: the instructions, and the per-topic doc paths +# an assistant is meant to read before writing any flashcard. +generate_llm_prompt() { + #============================================================================== + # Generate LLM Prompt for Anki Card Generation + #============================================================================== + echo -e "${YELLOW}Generating LLM prompt...${NC}" + + # Helper function to get doc link for a term + get_llm_doc_link() { + local term="$1" + local lang="$2" + # "true" if the term is a whole import line rather than a bare name. + # Defaulted for the same reason as the lookup helpers: every current + # caller passes it, but a bare "$3" would abort under `set -u`. + local is_import="${3:-false}" + + # Check if it's an internal/project-specific item + if [[ $term =~ ^@/ ]] || [[ $term =~ ^\./ ]] || [[ $term =~ ^app\. ]] || [[ $term =~ ^src/ ]] || [[ $term =~ from\ \'@/ ]] || [[ $term =~ from\ \'\./ ]]; then + echo "[INTERNAL - SKIP]" + return + fi + + # Try offline lookup + local offline_result + if [ "$is_import" = "true" ]; then + offline_result=$("$LOOKUP_SCRIPT" --import "$term" "$lang" 2>/dev/null | grep "^/" | head -1) + else + offline_result=$("$LOOKUP_SCRIPT" "$term" "$lang" 2>/dev/null | grep "^File:" | head -1 | sed 's/^File: //') + fi + + if [ -n "$offline_result" ]; then + echo "$offline_result" + else + echo "[NO OFFLINE DOC]" + fi + } + + # Generate keywords with doc links + generate_keywords_with_docs() { + local keywords_file="$RESULTS_DIR/grep_keywords.txt" + [ ! -f "$keywords_file" ] && echo "No keywords found" && return + + { grep -v '^#' "$keywords_file" || true; } | head -n "$TOP_N" | while read -r line; do + local count + count=$(echo "$line" | awk '{print $1}') + local keyword + keyword=$(echo "$line" | awk '{print $2}') + [ -z "$keyword" ] && continue + local doc_link + doc_link=$(get_llm_doc_link "$keyword" "$PRIMARY_LANG" "false") + echo "$count $keyword → $doc_link" + done + } + + # Generate functions with doc links + generate_functions_with_docs() { + local functions_file="$RESULTS_DIR/grep_function_calls.txt" + [ ! -f "$functions_file" ] && echo "No functions found" && return + + { grep -v '^#' "$functions_file" || true; } | head -n "$TOP_N" | while read -r line; do + local count + count=$(echo "$line" | awk '{print $1}') + local func + func=$(echo "$line" | awk '{print $2}') + + # Skip single-letter functions (minified code) or empty + if [ -z "$func" ] || [ ${#func} -le 1 ]; then + continue + fi + + local doc_link + doc_link=$(get_llm_doc_link "$func" "$PRIMARY_LANG" "false") + echo "$count $func() → $doc_link" + done + } + + # Generate imports with doc links + generate_imports_with_docs() { + local imports_file="$RESULTS_DIR/grep_imports.txt" + [ ! -f "$imports_file" ] && echo "No imports found" && return + + { grep -v '^#' "$imports_file" || true; } | head -n "$TOP_N" | while read -r line; do + local count + count=$(echo "$line" | awk '{print $1}') + local import_stmt + import_stmt=$(echo "$line" | cut -d' ' -f2-) + [ -z "$import_stmt" ] && continue + + # Check if internal import + if [[ $import_stmt =~ @/ ]] || [[ $import_stmt =~ \./ ]] || [[ $import_stmt =~ from\ app\. ]] || [[ $import_stmt =~ from\ src\. ]]; then + echo "$count $import_stmt → [INTERNAL - SKIP]" + else + local doc_link + doc_link=$(get_llm_doc_link "$import_stmt" "$PRIMARY_LANG" "true") + echo "$count $import_stmt → $doc_link" + fi + done + } + + cat >"$LLM_PROMPT_FILE" <<'PROMPT_HEADER' +# LLM Prompt: Generate Anki Flashcards + +You are creating Anki flashcards from code analysis. + +## CRITICAL INSTRUCTIONS + +1. **READ DOCS VIA TERMINAL** - Use the `cat` command to read each .md file: + ``` + cat /home/kuhy/.local/share/offline-docs/mdn-content/files/en-us/web/javascript/reference/statements/const/index.md + ``` +2. **DO NOT USE YOUR OWN KNOWLEDGE** - Base flashcards ONLY on the content you read from the files +3. **IF YOU CANNOT READ A FILE** - Report: "ERROR: Cannot read [path]" and skip that item +4. **NEVER FALL BACK TO GENERAL KNOWLEDGE** - If you can't read the file, skip it entirely +5. **READ ONE FILE AT A TIME** - Run cat for each topic before creating its flashcards + +PROMPT_HEADER + + cat >>"$LLM_PROMPT_FILE" <>"$LLM_PROMPT_FILE" <<'PROMPT_FOOTER' + +## Guidelines + +**CRITICAL - Keep answers EXTREMELY short:** +- Most answers should be **1-2 words** or **1 sentence** +- It's common and expected for an answer to be just: "Returns an array" or "Immutable" +- 2 sentences = longer answer, 3 sentences = absolute maximum (rare) +- Each flashcard tests ONE atomic piece of knowledge + +**NO DUPLICATES:** +- Before creating a card, check if you already created a similar question +- Each unique fact should appear in EXACTLY ONE card +- Do NOT create multiple cards asking the same thing with slightly different wording + +**What to include:** +- Concept cards: "What is X?" / "What does X do?" +- Syntax cards: "How do you write X?" (brief code snippet) +- Comparison cards: "X vs Y - what's the difference?" + +**What to SKIP (do NOT create cards for):** +- MDN frontmatter fields: title, slug, page-type, browser-compat, spec-urls +- YAML metadata between `---` markers at the start of files +- Any line that looks like metadata (key: value at start of doc) +- Empty answers - if you can't find content for the back, skip the card entirely + +**FINAL CARD FOR EACH TOPIC (EXCEPTION TO SHORT ANSWER RULE):** +- Add EXACTLY ONE full documentation card per topic (no duplicates!) +- Question: `[Topic] - Full MDN Documentation` +- Answer: Copy the .md file content STARTING AFTER the `---` frontmatter block +- Skip the YAML frontmatter (everything between the first two `---` lines) +- Do NOT create this card twice for the same topic + +**Skipped items - please review:** +- Items marked `[INTERNAL - SKIP]` are project-specific utilities - I skipped them +- Items marked `[NO OFFLINE DOC]` are third-party libraries without bundled docs +- If you want flashcards for skipped items, tell me which ones to include + +## OUTPUT: CREATE AN ANKI FILE + +**CREATE A FILE DIRECTLY** - Do not just output text. Use your file creation tool to create: + +**File path:** `~/.local/share/study-materials/anki_generated.txt` + +**Format:** Tab-separated values (TSV) with Anki metadata headers: + +``` +#separator:tab +#deck:CodeStudy::JavaScript +#notetype:CodeCard +#columns:Front Back Tags +What does const declare?Block-scoped variables with immutable bindings.javascript declarations +``` + +**Required headers at top of file:** +- `#separator:tab` - Specifies tab as delimiter +- `#deck:CodeStudy::[Language]` - Creates deck "CodeStudy" with sub-deck for language (e.g., CodeStudy::JavaScript) +- `#notetype:CodeCard` - Uses custom note type "CodeCard" (Anki will create if doesn't exist) +- `#columns:Front Back Tags` - Column headers (tab-separated) + +**Rules:** +- Use ACTUAL `` tags (not escaped <code>) +- Use `
` for line breaks within fields +- Use `
` for code blocks
+- Tags are space-separated
+- Escape any literal tabs within content as spaces
+
+**Example file content:**
+```
+#separator:tab
+#deck:CodeStudy::JavaScript
+#notetype:CodeCard
+#columns:Front	Back	Tags
+What does const declare?Block-scoped variables with immutable bindings.javascript declarations
+Can const be reassigned?No, throws TypeError.javascript declarations
+const - Full Documentation
[ENTIRE CONTENT OF const/index.md FILE]
javascript declarations full-doc +``` + +**After creating the file**, tell the user: +- File created at: ~/.local/share/study-materials/anki_generated.txt +- Import in Anki: File → Import → select the file +- Deck: CodeStudy::[Language], Note type: CodeCard +--- + +**Important:** +- Process only 5-10 items at a time to maintain quality +- Focus on items with offline documentation paths +- Output ONLY the TSV lines, no extra formatting or markdown +PROMPT_FOOTER + + echo -e "${GREEN}Created: $LLM_PROMPT_FILE${NC}" + +} diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/run_all.sh b/linux_configuration/scripts/single_use/utils/lib/tests/run_all.sh new file mode 100755 index 00000000..f337e23b --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/run_all.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# lib/tests/run_all.sh — run every generate_study_materials test in one process +# tree, and exit non-zero if any of them fails. +# +# Also the subject handed to shell_coverage.sh: coverage of a lib is only +# complete across the whole suite, since the doc-URL builders are exercised both +# directly and through get_doc_url's dispatch. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +failed=0 +for test_script in "${SCRIPT_DIR}"/test_study_*.sh; do + printf '\n### %s\n' "$(basename "$test_script")" + if ! "$test_script"; then + failed=$((failed + 1)) + fi +done + +if [[ $failed -gt 0 ]]; then + printf '\n%d test file(s) FAILED\n' "$failed" + exit 1 +fi + +printf '\nAll generate_study_materials test files passed.\n' diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/study_harness.sh b/linux_configuration/scripts/single_use/utils/lib/tests/study_harness.sh new file mode 100755 index 00000000..0449638d --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/study_harness.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# lib/tests/study_harness.sh — shared fixture for the generate_study_materials +# split. +# +# Sourced, not executed. These libs are almost pure string logic: the doc-URL +# builders map a term to a URL, and the only external process any of them runs +# is $LOOKUP_SCRIPT, which the tests point at a stub. Nothing here reaches the +# network or reads the real offline-docs mirror. +set -euo pipefail + +PASS=0 +FAIL=0 + +_t_pass() { + PASS=$((PASS + 1)) + printf ' OK: %s\n' "$1" +} + +_t_fail() { + FAIL=$((FAIL + 1)) + printf ' FAIL: %s\n' "$1" +} + +_t_eq() { + local want="$1" + local got="$2" + local what="$3" + if [[ "$got" == "$want" ]]; then + _t_pass "$what" + else + _t_fail "$what (want '${want}', got '${got}')" + fi +} + +# Assert a URL builder's output contains a substring. Most builders assemble a +# base URL plus the term, so matching the distinctive fragment is more robust +# than pinning the whole string, which would break on an upstream doc-site move +# without anything actually being wrong. +_t_has() { + local haystack="$1" + local needle="$2" + local what="$3" + if [[ "$haystack" == *"$needle"* ]]; then + _t_pass "$what" + else + _t_fail "$what (want a substring '${needle}', got '${haystack}')" + fi +} + +_t_summary() { + printf '\n%s: %d passed, %d failed\n' "$(basename "$0")" "$PASS" "$FAIL" + [[ $FAIL -eq 0 ]] +} + +TEST_TMPDIR="$(mktemp -d)" +trap 'rm -rf "${TEST_TMPDIR}"' EXIT + +# --- globals the libs read -------------------------------------------------- + +# Read by the generation phases, which the individual test files source +# rather than this harness -- so standalone shellcheck (as the hook runs it, +# with no -x) sees only the write here and reports SC2034. Exported because +# they genuinely cross into the sourced libs. +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export BLUE='\033[0;34m' +export YELLOW='\033[1;33m' +export NC='\033[0m' + +export RESULTS_DIR="${TEST_TMPDIR}/results" +export TOP_N=5 +export PRIMARY_LANG="python" +mkdir -p "${RESULTS_DIR}" + +export DOCS_FILE="${RESULTS_DIR}/documentation_links.md" +export ANKI_FILE="${RESULTS_DIR}/anki_cards.txt" +export LLM_PROMPT_FILE="${RESULTS_DIR}/llm_anki_prompt.md" + +export OFFLINE_DOCS_DIR="${TEST_TMPDIR}/offline-docs" +export LOOKUP_SCRIPT="${TEST_TMPDIR}/lookup_docs.sh" +export USE_OFFLINE_DOCS=false + +# Install a stub lookup_docs.sh and switch the libs onto the offline path. +# +# The real script prints "File: " for a term lookup and a bare path for +# an --import lookup, and lookup_offline parses each shape differently, so the +# stub reproduces both rather than one. +enable_offline_docs() { # [answer-path] + local answer="${1:-${OFFLINE_DOCS_DIR}/python/stdlib/os.md}" + mkdir -p "${OFFLINE_DOCS_DIR}" + cat >"${LOOKUP_SCRIPT}" <"${LOOKUP_SCRIPT}" + chmod +x "${LOOKUP_SCRIPT}" + USE_OFFLINE_DOCS=true +} + +# A mirror that answers ONLY for the given language and stays silent for every +# other. This is what exercises the TypeScript retry path: a TS lookup misses +# under "typescript", then succeeds when get_doc_url retries under "js". +enable_offline_docs_for_lang() { # [answer-path] + local only_lang="$1" + local answer="${2:-${OFFLINE_DOCS_DIR}/js/const.md}" + mkdir -p "${OFFLINE_DOCS_DIR}" + cat >"${LOOKUP_SCRIPT}" < ", which is what analyze_repo.sh emits. +stage_results() { + printf '%s\n' "42 def" "31 class" "20 import" >"${RESULTS_DIR}/grep_keywords.txt" + printf '%s\n' "18 print" "9 len" "4 range" >"${RESULTS_DIR}/grep_function_calls.txt" + printf '%s\n' "12 os" "7 sys" "3 json" >"${RESULTS_DIR}/grep_imports.txt" +} + +# Write the per-language analysis tree. generate_doc_links takes a completely +# different path when $RESULTS_DIR/per_language exists: instead of three global +# grep_*.txt files it walks keywords_.txt / functions_.txt / +# imports_.txt and resolves each language's terms against that language's +# docs, which is what makes a polyglot repo's index correct. +stage_per_language() { # ... + local dir="${RESULTS_DIR}/per_language" + mkdir -p "$dir" + local lang + for lang in "$@"; do + printf '%s\n' "9 alpha_${lang}" "8 beta_${lang}" >"${dir}/keywords_${lang}.txt" + printf '%s\n' "7 fn_${lang}" >"${dir}/functions_${lang}.txt" + printf '%s\n' "6 mod_${lang}" >"${dir}/imports_${lang}.txt" + done +} + +# tokei output in the shape detect_language parses. +stage_tokei() { # [second-language] + { + printf '===============================================================================\n' + printf ' Language Files Lines Code Comments Blanks\n' + printf '===============================================================================\n' + printf ' %-18s 42 5120 4001 610 509\n' "$1" + [[ -n "${2:-}" ]] && printf ' %-18s 12 1400 1100 150 150\n' "$2" + printf '===============================================================================\n' + } >"${RESULTS_DIR}/tokei_stats.txt" +} + +reset_state() { + rm -rf "${RESULTS_DIR:?}" + mkdir -p "${RESULTS_DIR}" + TOP_N=5 + PRIMARY_LANG="python" + disable_offline_docs +} +reset_state diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_lookup.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_lookup.sh new file mode 100755 index 00000000..836404d1 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_lookup.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Tests for lib/study_doc_lookup.sh — offline lookup, language dispatch and +# language detection. +# +# The interesting behaviour is the preference order: when an offline mirror is +# installed, a term should resolve to a local FILE PATH an assistant can read +# with `cat`, and only fall back to an online URL when the mirror has no answer. +# That distinction is the whole point of the offline path — a card citing a URL +# the assistant cannot fetch is a card it will answer from memory instead. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=study_harness.sh +. "${SCRIPT_DIR}/study_harness.sh" +# shellcheck source=../study_doc_urls.sh +. "${SCRIPT_DIR}/../study_doc_urls.sh" +# shellcheck source=../study_doc_urls_more.sh +. "${SCRIPT_DIR}/../study_doc_urls_more.sh" +# shellcheck source=../study_doc_lookup.sh +. "${SCRIPT_DIR}/../study_doc_lookup.sh" + +echo "== lookup_offline: disabled ==" +reset_state +if lookup_offline "os" python "" >/dev/null 2>&1; then + _t_fail "with offline docs off, lookup_offline fails" +else + _t_pass "with offline docs off, lookup_offline fails" +fi + +echo "== lookup_offline: a term lookup ==" +reset_state +enable_offline_docs "${OFFLINE_DOCS_DIR}/python/os.md" +_t_eq "${OFFLINE_DOCS_DIR}/python/os.md" "$(lookup_offline "os" python "")" \ + "a term resolves to the local file path" + +echo "== lookup_offline: an import-aware lookup ==" +reset_state +enable_offline_docs "${OFFLINE_DOCS_DIR}/js/array.md" +_t_eq "${OFFLINE_DOCS_DIR}/js/array.md" "$(lookup_offline "" javascript "import { x } from 'y'")" \ + "an import line resolves through the --import path" + +echo "== lookup_offline: mirror has no answer ==" +reset_state +enable_offline_docs_empty +if lookup_offline "frobnicate" python "" >/dev/null 2>&1; then + _t_fail "an unanswered term fails rather than returning empty" +else + _t_pass "an unanswered term fails rather than returning empty" +fi + +echo "== get_doc_url: dispatches per language ==" +reset_state +_t_has "$(get_doc_url "def" python "")" "docs.python.org" "python dispatches to python_doc_url" +_t_has "$(get_doc_url "const" javascript "")" "developer.mozilla.org" "javascript dispatches to js_doc_url" +_t_has "$(get_doc_url "interface" typescript "")" "typescriptlang.org" "typescript dispatches to ts_doc_url" +_t_has "$(get_doc_url "printf" c "")" "cppreference.com" "c dispatches to c_doc_url" +_t_has "$(get_doc_url "vector" cpp "")" "cppreference.com" "cpp dispatches to cpp_doc_url" +_t_has "$(get_doc_url "fn" rust "")" "rust-lang.org" "rust dispatches to rust_doc_url" +_t_has "$(get_doc_url "func" go "")" "go.dev" "go dispatches to go_doc_url" +_t_has "$(get_doc_url "def" ruby "")" "ruby-doc.org" "ruby dispatches to ruby_doc_url" +_t_has "$(get_doc_url "class" java "")" "oracle.com" "java dispatches to java_doc_url" +_t_has "$(get_doc_url "if" shell "")" "gnu.org" "shell dispatches to shell_doc_url" + +echo "== get_doc_url: an unknown language still answers ==" +reset_state +out="$(get_doc_url "something" brainfuck "")" +if [[ -n "$out" ]]; then + _t_pass "an unrecognised language still produces a link" +else + _t_fail "an unrecognised language still produces a link (returned nothing)" +fi + +echo "== get_doc_url: prefers the offline mirror when it has an answer ==" +reset_state +enable_offline_docs "${OFFLINE_DOCS_DIR}/python/os.md" +out="$(get_doc_url "os" python "")" +_t_has "$out" "${OFFLINE_DOCS_DIR}" "an offline hit wins over the online URL" + +echo "== get_doc_url: falls back online when the mirror misses ==" +reset_state +enable_offline_docs_empty +_t_has "$(get_doc_url "os" python "")" "docs.python.org" "an offline miss falls back to the online URL" + +echo "== get_doc_url: TypeScript borrows the JavaScript offline mirror ==" +reset_state +# The mirror answers for "js" and nothing else, which is the real shape: an +# offline-docs install carries MDN, not a separate TypeScript tree. A TS lookup +# therefore misses under "typescript" and only succeeds on the retry under +# "js". Without that retry a TS repo would silently never use the docs it has. +enable_offline_docs_for_lang js "${OFFLINE_DOCS_DIR}/js/const.md" +_t_has "$(get_doc_url "const" typescript "")" "${OFFLINE_DOCS_DIR}" \ + "a TS term resolves through the JS mirror" + +echo "== get_doc_url: a TS runtime term falls through to the JS builder ==" +reset_state +_t_has "$(get_doc_url "const" typescript "")" "developer.mozilla.org" \ + "a runtime term prefers MDN over the TS handbook" +_t_has "$(get_doc_url "interface" typescript "")" "typescriptlang.org" \ + "a TS-only term still gets the TS handbook" + +echo "== detect_language ==" +reset_state +stage_tokei "Python" "JavaScript" +_t_eq "python" "$(detect_language)" "the language with the most code wins" + +reset_state +stage_tokei "Rust" +_t_eq "rust" "$(detect_language)" "a single-language repo detects that language" + +reset_state +stage_tokei "C++" +out="$(detect_language)" +if [[ -n "$out" ]]; then + _t_pass "a punctuated language name still maps to something ('${out}')" +else + _t_fail "a punctuated language name still maps to something" +fi + +echo "== detect_language: no stats file ==" +reset_state +out="$(detect_language)" +if [[ -n "$out" ]]; then + _t_pass "a missing tokei_stats.txt falls back to a default ('${out}')" +else + _t_fail "a missing tokei_stats.txt falls back to a default" +fi + +_t_summary diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_urls.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_urls.sh new file mode 100755 index 00000000..0a25e86f --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_urls.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Tests for lib/study_doc_urls.sh — the Python, JS, TS and C/C++ doc-URL +# builders. +# +# Each builder is a case dispatch over term categories with a search-URL +# fallback, so every arm gets one assertion plus one for the fallback. The +# fallback matters more than it looks: it is what every unrecognised term hits, +# so a card for an unknown identifier still links somewhere useful instead of +# to an empty string. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=study_harness.sh +. "${SCRIPT_DIR}/study_harness.sh" +# shellcheck source=../study_doc_urls.sh +. "${SCRIPT_DIR}/../study_doc_urls.sh" + +echo "== python_doc_url ==" +_t_eq "https://docs.python.org/3/reference/compound_stmts.html" \ + "$(python_doc_url "def" keyword)" "a keyword links to the statements reference" +_t_eq "https://docs.python.org/3/library/functions.html#print" \ + "$(python_doc_url "print" builtin)" "a builtin links to its anchor in functions.html" +_t_eq "https://docs.python.org/3/library/os.html" \ + "$(python_doc_url "os" module)" "a stdlib module links to its own page" +_t_eq "https://docs.python.org/3/library/unittest.mock.html" \ + "$(python_doc_url "MagicMock" module)" "a mock helper links to unittest.mock" +_t_eq "https://docs.python.org/3/search.html?q=frobnicate" \ + "$(python_doc_url "frobnicate" keyword)" "an unknown term falls back to search" + +echo "== js_doc_url ==" +_t_has "$(js_doc_url "const" keyword)" "developer.mozilla.org" "a JS keyword links to MDN" +_t_has "$(js_doc_url "Array" builtin)" "Array" "a builtin object names itself in the URL" +_t_has "$(js_doc_url "frobnicate" keyword)" "frobnicate" "an unknown JS term still carries the term" + +echo "== ts_doc_url ==" +_t_has "$(ts_doc_url "interface" keyword)" "typescriptlang.org" "a TS term links to the TS handbook" +_t_has "$(ts_doc_url "Partial" keyword)" "utility-types" "a TS utility type links to the utility-types page" +# A term TypeScript does not define is a runtime feature, so it deliberately +# falls through to the JS builder rather than to a TS search page. +_t_has "$(ts_doc_url "frobnicate" keyword)" "developer.mozilla.org" "an unknown TS term falls through to MDN" + +echo "== js_doc_url: every category of builtin ==" +_t_has "$(js_doc_url "map" builtin)" "Global_Objects/Array" "an array method links under Array" +_t_has "$(js_doc_url "split" builtin)" "Global_Objects/String" "a string method links under String" +_t_has "$(js_doc_url "then" builtin)" "Global_Objects/Promise" "a promise method links under Promise" +_t_has "$(js_doc_url "fetch" builtin)" "docs/Web/API" "a browser API links under Web/API" + +echo "== c_doc_url ==" +_t_has "$(c_doc_url "printf" builtin)" "cppreference.com" "a C library function links to cppreference" +_t_has "$(c_doc_url "for" keyword)" "cppreference.com" "a C keyword links to cppreference" +_t_has "$(c_doc_url "frobnicate" keyword)" "cppreference.com" "an unknown C term stays on cppreference" + +_t_has "$(c_doc_url "stdio" module)" "c/header/stdio.h" "a C header links to its header page" +_t_has "$(c_doc_url "malloc" builtin)" "c/memory" "an allocator links to the memory page" +_t_has "$(c_doc_url "strlen" builtin)" "c/string/byte" "a string function links to the byte-string page" + +echo "== cpp_doc_url ==" +_t_has "$(cpp_doc_url "vector" builtin)" "cppreference.com" "a C++ container links to cppreference" +_t_has "$(cpp_doc_url "class" keyword)" "cppreference.com" "a C++ keyword links to cppreference" +_t_has "$(cpp_doc_url "frobnicate" keyword)" "cppreference.com" "an unknown C++ term stays on cppreference" + +_t_has "$(cpp_doc_url "sort" builtin)" "cpp/algorithm/sort" "an algorithm links under cpp/algorithm" +_t_has "$(cpp_doc_url "unique_ptr" builtin)" "cpp/memory/unique_ptr" "a smart pointer links under cpp/memory" +_t_has "$(cpp_doc_url "optional" builtin)" "cpp/utility" "a utility type links under cpp/utility" + +echo "== every builder answers for every input ==" +# The generation phases pipe whatever the analysis found straight into these, +# including punctuation and empty strings from a malformed grep line. A builder +# that returned nothing would silently produce a card with no link at all, so +# the invariant worth pinning is that output is never empty. +for fn in python_doc_url js_doc_url ts_doc_url c_doc_url cpp_doc_url; do + for term in "" "x" "std::vector" "__init__" "a-b" "3"; do + out="$("$fn" "$term" keyword)" + if [[ -n "$out" ]]; then + _t_pass "${fn} answers for '${term}'" + else + _t_fail "${fn} answers for '${term}' (returned nothing)" + fi + done +done + +_t_summary diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_urls_more.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_urls_more.sh new file mode 100755 index 00000000..46883e09 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_doc_urls_more.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tests for lib/study_doc_urls_more.sh — the Rust, Go, Ruby, Java and shell +# doc-URL builders. +# +# Same shape as test_study_doc_urls.sh: one assertion per case arm plus the +# fallback, since an unmatched term is the common case for anything the +# analysis extracted that is project-specific rather than language-level. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=study_harness.sh +. "${SCRIPT_DIR}/study_harness.sh" +# shellcheck source=../study_doc_urls_more.sh +. "${SCRIPT_DIR}/../study_doc_urls_more.sh" + +echo "== rust_doc_url ==" +_t_has "$(rust_doc_url "fn" keyword)" "doc.rust-lang.org" "a Rust keyword links to the Rust docs" +_t_has "$(rust_doc_url "Option" builtin)" "Option" "a std type names itself" +_t_has "$(rust_doc_url "Clone" builtin)" "Clone" "a derivable trait names itself" +_t_has "$(rust_doc_url "println" builtin)" "macro" "a macro links to its macro page" +_t_has "$(rust_doc_url "frobnicate" keyword)" "doc.rust-lang.org" "an unknown Rust term stays on the Rust docs" + +echo "== go_doc_url ==" +_t_has "$(go_doc_url "func" keyword)" "go.dev" "a Go keyword links to go.dev" +_t_has "$(go_doc_url "append" builtin)" "builtin" "a builtin links to the builtin package" +_t_has "$(go_doc_url "fmt" module)" "fmt" "a stdlib package names itself" +_t_has "$(go_doc_url "frobnicate" keyword)" "go.dev" "an unknown Go term stays on go.dev" + +echo "== ruby_doc_url ==" +_t_has "$(ruby_doc_url "def" keyword)" "ruby-doc.org" "a Ruby keyword links to ruby-doc.org" +_t_has "$(ruby_doc_url "String" builtin)" "String" "a core class names itself" +_t_has "$(ruby_doc_url "each" builtin)" "Enumerable" "an Enumerable method links to the Enumerable page" +_t_has "$(ruby_doc_url "frobnicate" keyword)" "ruby-doc.org" "an unknown Ruby term stays on ruby-doc.org" + +echo "== java_doc_url ==" +_t_has "$(java_doc_url "class" keyword)" "oracle.com" "a Java keyword links to the Oracle docs" +_t_has "$(java_doc_url "String" builtin)" "String" "a core class names itself" +_t_has "$(java_doc_url "frobnicate" keyword)" "oracle.com" "an unknown Java term stays on the Oracle docs" + +echo "== shell_doc_url ==" +_t_has "$(shell_doc_url "if" keyword)" "gnu.org" "a shell keyword links to the bash manual" +_t_has "$(shell_doc_url "declare" builtin)" "gnu.org" "a shell builtin links to the bash manual" +# External commands get a man page rather than the bash manual, because they +# are not part of the shell -- a card saying otherwise would be wrong. +_t_has "$(shell_doc_url "grep" builtin)" "man" "an external command links to its man page" +_t_has "$(shell_doc_url "frobnicate" keyword)" "man" "an unknown command is treated as external" + +echo "== every builder answers for every input ==" +# These are fed straight from the analysis output, punctuation and all. A +# builder returning nothing would emit a card with no link, which is worse than +# a link to a search page. +for fn in rust_doc_url go_doc_url ruby_doc_url java_doc_url shell_doc_url; do + for term in "" "x" "a-b" "3" "__weird__"; do + out="$("$fn" "$term" keyword)" + if [[ -n "$out" ]]; then + _t_pass "${fn} answers for '${term}'" + else + _t_fail "${fn} answers for '${term}' (returned nothing)" + fi + done +done + +_t_summary diff --git a/linux_configuration/scripts/single_use/utils/lib/tests/test_study_gen.sh b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_gen.sh new file mode 100755 index 00000000..5b86e770 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/lib/tests/test_study_gen.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# Tests for the three generation phases: lib/study_gen_docs.sh, +# lib/study_gen_anki.sh and lib/study_gen_llm.sh. +# +# Each phase reads the grep_*.txt files the analysis step wrote and produces +# one output file. The cases below pin what actually matters about each output +# rather than its exact bytes: that every extracted term appears, that the Anki +# file keeps the tab-separated shape its importer requires, and that the LLM +# prompt keeps the instructions that stop an assistant answering from memory +# instead of from the docs it was told to read. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=study_harness.sh +. "${SCRIPT_DIR}/study_harness.sh" +# shellcheck source=../study_doc_urls.sh +. "${SCRIPT_DIR}/../study_doc_urls.sh" +# shellcheck source=../study_doc_urls_more.sh +. "${SCRIPT_DIR}/../study_doc_urls_more.sh" +# shellcheck source=../study_doc_lookup.sh +. "${SCRIPT_DIR}/../study_doc_lookup.sh" +# shellcheck source=../study_gen_docs.sh +. "${SCRIPT_DIR}/../study_gen_docs.sh" +# shellcheck source=../study_gen_anki.sh +. "${SCRIPT_DIR}/../study_gen_anki.sh" +# shellcheck source=../study_gen_llm.sh +. "${SCRIPT_DIR}/../study_gen_llm.sh" + +echo "== generate_doc_links: writes a section per term kind ==" +reset_state +stage_results +generate_doc_links >/dev/null +docs="$(cat "${DOCS_FILE}")" +_t_has "$docs" "def" "a keyword appears in the doc index" +_t_has "$docs" "print" "a function call appears" +_t_has "$docs" "os" "an import appears" +_t_has "$docs" "docs.python.org" "terms carry a resolved documentation link" + +echo "== generate_doc_links: honours TOP_N ==" +reset_state +printf '%s\n' "9 alpha" "8 bravo" "7 charlie" "6 delta" >"${RESULTS_DIR}/grep_keywords.txt" +: >"${RESULTS_DIR}/grep_function_calls.txt" +: >"${RESULTS_DIR}/grep_imports.txt" +TOP_N=2 +generate_doc_links >/dev/null +docs="$(cat "${DOCS_FILE}")" +_t_has "$docs" "alpha" "the top term is included" +if [[ "$docs" == *"charlie"* ]]; then + _t_fail "a term past TOP_N is excluded" +else + _t_pass "a term past TOP_N is excluded" +fi + +echo "== generate_doc_links: missing input files are not fatal ==" +reset_state +# A repo with no imports at all is normal, and the analysis simply does not +# write that file. The phase must still produce its other sections. +printf '%s\n' "9 def" >"${RESULTS_DIR}/grep_keywords.txt" +generate_doc_links >/dev/null +_t_has "$(cat "${DOCS_FILE}")" "def" "the surviving section is still written" + +echo "== generate_doc_links: uses the offline mirror when installed ==" +reset_state +stage_results +enable_offline_docs "${OFFLINE_DOCS_DIR}/python/os.md" +generate_doc_links >/dev/null +_t_has "$(cat "${DOCS_FILE}")" "${OFFLINE_DOCS_DIR}" "cards cite a local path an assistant can read" + +echo "== generate_doc_links: per-language mode ==" +reset_state +stage_results +# The presence of per_language/ switches the phase onto an entirely separate +# path that resolves each language's terms against that language's docs. +stage_per_language python javascript rust +generate_doc_links >/dev/null +docs="$(cat "${DOCS_FILE}")" +_t_has "$docs" "Language Keywords" "the per-language section header is written" +_t_has "$docs" "Python" "a language gets a display name" +_t_has "$docs" "JavaScript" "the JavaScript display name is mapped" +_t_has "$docs" "Rust" "a third language is included" +_t_has "$docs" "alpha_python" "a per-language keyword is listed" +_t_has "$docs" "fn_javascript" "a per-language function is listed" +_t_has "$docs" "mod_rust" "a per-language import is listed" + +echo "== generate_doc_links: per-language name mapping ==" +reset_state +stage_results +stage_per_language c_cpp typescript shell +generate_doc_links >/dev/null +docs="$(cat "${DOCS_FILE}")" +_t_has "$docs" "C/C++" "c_cpp maps to the C/C++ display name" +_t_has "$docs" "TypeScript" "typescript maps to its display name" +_t_has "$docs" "Shell/Bash" "shell maps to the Shell/Bash display name" + +echo "== generate_doc_links: per-language covers the remaining names ==" +reset_state +stage_results +stage_per_language go ruby java klingon +generate_doc_links >/dev/null +docs="$(cat "${DOCS_FILE}")" +_t_has "$docs" "Go" "go maps to its display name" +_t_has "$docs" "Ruby" "ruby maps to its display name" +_t_has "$docs" "Java" "java maps to its display name" +_t_has "$docs" "klingon" "an unmapped language falls back to its raw name" + +echo "== generate_doc_links: per-language skips empty and comment lines ==" +reset_state +stage_results +stage_per_language python +# An empty file and a comment-only line are both normal analysis output; each +# must be skipped rather than producing a card with no term. +: >"${RESULTS_DIR}/per_language/functions_python.txt" +printf '%s\n' "# a comment" "5 realterm" >"${RESULTS_DIR}/per_language/imports_python.txt" +generate_doc_links >/dev/null +docs="$(cat "${DOCS_FILE}")" +_t_has "$docs" "realterm" "the real term survives" +if [[ "$docs" == *"a comment"* ]]; then + _t_fail "a comment line is skipped" +else + _t_pass "a comment line is skipped" +fi + +echo "== generate_anki_cards: tab-separated with a header ==" +reset_state +stage_results +generate_anki_cards >/dev/null +anki="$(cat "${ANKI_FILE}")" +_t_has "$anki" "Anki Import File" "the importer header is present" +_t_has "$anki" "Fields separated by: Tab" "the header states the separator" +_t_has "$anki" "def" "a keyword card is written" +_t_has "$anki" "print" "a function card is written" +# Every non-comment line must carry at least one tab, or the import silently +# collapses the card into a single field. +bad=0 +while IFS= read -r line; do + [[ -z "$line" || "$line" == \#* ]] && continue + [[ "$line" == *$'\t'* ]] || bad=$((bad + 1)) +done <"${ANKI_FILE}" +_t_eq "0" "$bad" "every card line is tab-separated" + +echo "== generate_anki_cards: honours TOP_N ==" +reset_state +printf '%s\n' "9 alpha" "8 bravo" "7 charlie" >"${RESULTS_DIR}/grep_keywords.txt" +: >"${RESULTS_DIR}/grep_function_calls.txt" +TOP_N=1 +generate_anki_cards >/dev/null +anki="$(cat "${ANKI_FILE}")" +_t_has "$anki" "alpha" "the top term becomes a card" +if [[ "$anki" == *"bravo"* ]]; then + _t_fail "a term past TOP_N is excluded" +else + _t_pass "a term past TOP_N is excluded" +fi + +echo "== generate_anki_cards: no input files still writes a valid file ==" +reset_state +generate_anki_cards >/dev/null +_t_has "$(cat "${ANKI_FILE}")" "Anki Import File" "an empty analysis still produces an importable file" + +echo "== generate_anki_cards: one card template per keyword category ==" +reset_state +# Each category gets its own answer text, so a card for `if` explains control +# flow rather than repeating a generic definition. Covering every arm is what +# proves no category silently falls through to the default template. +printf '%s\n' "9 if" "8 for" "7 try" "6 class" "5 async" "4 frobnicate" \ + >"${RESULTS_DIR}/grep_keywords.txt" +: >"${RESULTS_DIR}/grep_function_calls.txt" +TOP_N=6 +generate_anki_cards >/dev/null +anki="$(cat "${ANKI_FILE}")" +_t_has "$anki" "Conditional control flow" "a conditional keyword gets the control-flow answer" +_t_has "$anki" "Loop construct" "a loop keyword gets the loop answer" +_t_has "$anki" "Exception handling" "an exception keyword gets the exception answer" +_t_has "$anki" "Type definition" "a type keyword gets the type answer" +_t_has "$anki" "Asynchronous programming" "an async keyword gets the async answer" +_t_has "$anki" "frobnicate" "an uncategorised keyword still gets a card" + +echo "== generate_llm_prompt: project-internal items are skipped ==" +reset_state +# A relative or aliased import points at the repo being studied, not at any +# documentation, so a flashcard for it could only be answered from guesswork. +# These must be marked skippable rather than sent to the assistant as a topic. +printf '%s\n' "9 ./local_module" "8 @/aliased" "7 src/thing" "6 app.config" "5 os" \ + >"${RESULTS_DIR}/grep_imports.txt" +printf '%s\n' "9 def" >"${RESULTS_DIR}/grep_keywords.txt" +printf '%s\n' "8 print" >"${RESULTS_DIR}/grep_function_calls.txt" +TOP_N=5 +generate_llm_prompt >/dev/null +_t_has "$(cat "${LLM_PROMPT_FILE}")" "INTERNAL - SKIP" "an internal import is marked skippable" + +echo "== generate_llm_prompt: single-letter functions are dropped ==" +reset_state +# Minified JS turns every function into one or two characters. A flashcard for +# `a` teaches nothing and cannot be looked up, so those are dropped rather than +# sent to the assistant as topics. +printf '%s\n' "9 def" >"${RESULTS_DIR}/grep_keywords.txt" +printf '%s\n' "9 a" "8 b" "7 realFunction" >"${RESULTS_DIR}/grep_function_calls.txt" +printf '%s\n' "6 os" >"${RESULTS_DIR}/grep_imports.txt" +TOP_N=5 +generate_llm_prompt >/dev/null +prompt="$(cat "${LLM_PROMPT_FILE}")" +_t_has "$prompt" "realFunction" "a real function name survives" +if [[ "$prompt" =~ [0-9]+\ a$'\n' ]]; then + _t_fail "a single-letter function is dropped" +else + _t_pass "a single-letter function is dropped" +fi + +echo "== generate_llm_prompt: keeps the read-the-docs instructions ==" +reset_state +stage_results +generate_llm_prompt >/dev/null +prompt="$(cat "${LLM_PROMPT_FILE}")" +# These four instructions are the entire reason the prompt exists: without +# them an assistant writes flashcards from memory, which is exactly the +# failure the offline-docs pipeline was built to prevent. +_t_has "$prompt" "READ DOCS VIA TERMINAL" "the read-the-file instruction survives" +_t_has "$prompt" "DO NOT USE YOUR OWN KNOWLEDGE" "the no-memory instruction survives" +_t_has "$prompt" "IF YOU CANNOT READ A FILE" "the unreadable-file instruction survives" +_t_has "$prompt" "NEVER FALL BACK TO GENERAL KNOWLEDGE" "the no-fallback instruction survives" +_t_has "$prompt" "def" "the extracted keywords are listed" + +echo "== generate_llm_prompt: import lines are handled as imports ==" +reset_state +stage_results +printf '%s\n' "12 import os" "7 from sys import argv" >"${RESULTS_DIR}/grep_imports.txt" +generate_llm_prompt >/dev/null +_t_has "$(cat "${LLM_PROMPT_FILE}")" "os" "a whole import statement still resolves to its module" + +echo "== generate_llm_prompt: offline paths are cited when available ==" +reset_state +stage_results +enable_offline_docs "${OFFLINE_DOCS_DIR}/python/os.md" +generate_llm_prompt >/dev/null +_t_has "$(cat "${LLM_PROMPT_FILE}")" "${OFFLINE_DOCS_DIR}" "the prompt cites readable local paths" + +echo "== generate_llm_prompt: no inputs still writes the instructions ==" +reset_state +generate_llm_prompt >/dev/null +_t_has "$(cat "${LLM_PROMPT_FILE}")" "DO NOT USE YOUR OWN KNOWLEDGE" \ + "an empty analysis still produces a usable prompt" + +_t_summary