testsAndMisc/python_pkg/code_tutor/_analyzer.py

"""Language-agnostic code item extraction from source files.

Supports Python via the ``ast`` module (exact line ranges) and other languages
via a regex heuristic.  Binary files and vendored directories are skipped.
"""

from __future__ import annotations

import ast
from dataclasses import dataclass, field
from pathlib import Path

from python_pkg.code_tutor._scan_tables import (
    _FUNCTION_RE,
    _OTHER_LANGS,
    _SKIP_DIRS,
    _SKIP_SUFFIXES,
)


@dataclass
class CodeItem:
    """A single extractable code unit (function or method).

    Attributes:
        id: Dotted identifier, e.g. ``module.submodule.function_name``.
        file: Relative path from the codebase root.
        type: ``"function"`` or ``"async_function"``.
        name: Bare function or method name.
        start_line: 1-based first line of the definition.
        end_line: 1-based last line of the definition.
        class_name: Enclosing class name, or ``""`` for module-level functions.
        depends_on: IDs of items that must be understood first.
    """

    id: str
    file: str
    type: str
    name: str
    start_line: int
    end_line: int
    class_name: str = ""
    depends_on: list[str] = field(default_factory=list)


def _is_binary(path: Path) -> bool:
    """Return True if *path* appears to be a binary file (null-byte heuristic).

    Uses the same heuristic as ``git`` and GNU ``grep``: a file is binary if
    its first 512 bytes contain a null byte.  Raises ``OSError`` on read
    failure so callers can handle unreadable files separately.

    Args:
        path: File to inspect.

    Returns:
        True when the first 512 bytes contain a null byte; False otherwise.

    Raises:
        OSError: When the file cannot be read.
    """
    chunk = path.read_bytes()[:512]
    return b"\x00" in chunk


def _should_skip(rel: str) -> bool:
    """Return True if this relative path should be excluded from analysis.

    Args:
        rel: Path relative to the codebase root.

    Returns:
        True when any path component is in ``_SKIP_DIRS`` or the file
        extension is in ``_SKIP_SUFFIXES``.
    """
    parts = Path(rel).parts
    if any(p in _SKIP_DIRS for p in parts):
        return True
    return Path(rel).suffix.lower() in _SKIP_SUFFIXES


def _make_id(rel_path: str, name: str) -> str:
    """Build a dotted item identifier from a relative path and function name.

    Args:
        rel_path: Path relative to the codebase root.
        name: Function or method name.

    Returns:
        Dotted identifier like ``module.submodule.function_name``.
    """
    without_ext = Path(rel_path).with_suffix("")
    return ".".join(without_ext.parts) + "." + name


class _FunctionVisitor(ast.NodeVisitor):
    """AST visitor that extracts function/method items with class context.

    Args:
        rel_path: Relative path of the file being visited.
    """

    def __init__(self, rel_path: str) -> None:
        """Initialise with file path and empty state."""
        self._rel_path = rel_path
        self._class_stack: list[str] = []
        self.items: list[CodeItem] = []

    def visit_ClassDef(self, node: ast.ClassDef) -> None:
        """Push class name onto stack, recurse, then pop."""
        self._class_stack.append(node.name)
        self.generic_visit(node)
        self._class_stack.pop()

    def _add(self, node: ast.FunctionDef | ast.AsyncFunctionDef, kind: str) -> None:
        end = getattr(node, "end_lineno", node.lineno)
        self.items.append(
            CodeItem(
                id=_make_id(self._rel_path, node.name),
                file=self._rel_path,
                type=kind,
                name=node.name,
                start_line=node.lineno,
                end_line=end,
                class_name=self._class_stack[-1] if self._class_stack else "",
            )
        )

    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
        """Record function and recurse into body."""
        self._add(node, "function")
        self.generic_visit(node)

    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
        """Record async function and recurse into body."""
        self._add(node, "async_function")
        self.generic_visit(node)


def _extract_python(path: Path, rel_path: str) -> list[CodeItem]:
    """Extract function/method items from a Python file using the AST.

    Args:
        path: Absolute path to the Python file.
        rel_path: Relative path used to build item IDs.

    Returns:
        List of ``CodeItem`` instances, one per function definition found.
        Returns an empty list on ``SyntaxError``.
    """
    try:
        source = path.read_text(encoding="utf-8", errors="replace")
        tree = ast.parse(source)
    except SyntaxError:
        return []

    visitor = _FunctionVisitor(rel_path)
    visitor.visit(tree)
    return visitor.items


def _extract_other(path: Path, rel_path: str) -> list[CodeItem]:
    """Extract function-like items from non-Python files via regex.

    Scans each line for a function-definition keyword followed by an
    identifier and opening parenthesis.  End line is estimated at
    ``start + 40`` (clamped to file length).

    Args:
        path: Absolute path to the source file.
        rel_path: Relative path used to build item IDs.

    Returns:
        List of ``CodeItem`` instances.  Returns an empty list on ``OSError``.
    """
    try:
        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
    except OSError:
        return []

    items: list[CodeItem] = []
    for lineno, line in enumerate(lines, start=1):
        match = _FUNCTION_RE.match(line)
        if match:
            name = match.group(1)
            end = min(lineno + 40, len(lines))
            items.append(
                CodeItem(
                    id=_make_id(rel_path, name),
                    file=rel_path,
                    type="function",
                    name=name,
                    start_line=lineno,
                    end_line=end,
                )
            )
    return items


def extract_items(codebase: Path) -> list[CodeItem]:
    """Walk *codebase* and return all extractable code items.

    Args:
        codebase: Root directory to analyse.

    Returns:
        Flat list of ``CodeItem`` instances, unsorted.
    """
    items: list[CodeItem] = []
    for path in sorted(codebase.rglob("*")):
        if not path.is_file():
            continue
        rel = str(path.relative_to(codebase))
        if _should_skip(rel):
            continue
        try:
            binary = _is_binary(path)
        except OSError:
            continue
        if binary:
            continue
        if path.suffix == ".py":
            items.extend(_extract_python(path, rel))
        elif path.suffix in _OTHER_LANGS:
            items.extend(_extract_other(path, rel))
    return items