From 0976c84254f39a87e95985a8116ed5a69f0cc0b3 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 11 Jul 2026 17:58:49 +0200 Subject: [PATCH] Add CI-mirror pre-push gate + requirements-sync check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the CI-green safeguards rolled out repo-wide (see screen-locker): - scripts/check_requirements_sync.py: pre-commit hook that fails if a package import isn't declared in requirements.txt. On rollout it caught a real latent gap — diet_guard/_mcp.py imports pydantic but it was only present transitively (via mcp); now declared explicitly. - scripts/ci_mirror.sh: pre-push gate = requirements.txt-only venv (hash-gated) running `pre-commit run --all-files` + `pytest`, replacing the plain pytest-coverage pre-push hook. Only the tooling files + the pydantic declaration are committed; unrelated in-flight work in the tree is left untouched. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_018PMRRVVkeRyrSaaqueMZKG --- .gitignore | 3 + .pre-commit-config.yaml | 14 ++- requirements.txt | 1 + scripts/check_requirements_sync.py | 196 +++++++++++++++++++++++++++++ scripts/ci_mirror.sh | 93 ++++++++++++++ 5 files changed, 303 insertions(+), 4 deletions(-) create mode 100755 scripts/check_requirements_sync.py create mode 100755 scripts/ci_mirror.sh diff --git a/.gitignore b/.gitignore index 22de16e..8c80821 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ coverage.lcov htmlcov/ *.log .DS_Store + +# ci_mirror.sh cached requirements.txt venv (rebuilt on demand) +.ci-mirror-venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 323ca0d..7f65ea8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,12 @@ repos: entry: '(?i)#\s*ruff:\s*noqa' language: pygrep types: [python] + - id: requirements-sync + name: requirements.txt covers all package imports + entry: python3 scripts/check_requirements_sync.py + language: system + pass_filenames: false + files: ^(diet_guard/.*\.py|requirements\.txt)$ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.2 @@ -115,12 +121,12 @@ repos: - repo: local hooks: - - id: pytest-coverage - name: pytest with coverage enforcement - entry: python -m pytest + - id: ci-mirror + name: CI mirror (clean-venv pytest + pre-commit --all-files) + entry: bash scripts/ci_mirror.sh language: system - types: [python] pass_filenames: false + always_run: true require_serial: true stages: [pre-push] diff --git a/requirements.txt b/requirements.txt index 1949d56..268e520 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ gatelock @ git+https://github.com/kuhyx/utils@gatelock-v0.1.0#subdirectory=gatel mcp>=1.9 mypy>=1.8.0 pre-commit>=3.6.0 +pydantic>=2 pylint>=3.0.0 pytest>=8.0.0 pytest-cov>=4.1.0 diff --git a/scripts/check_requirements_sync.py b/scripts/check_requirements_sync.py new file mode 100755 index 0000000..ef61b29 --- /dev/null +++ b/scripts/check_requirements_sync.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Fail if the package imports a third-party dist not declared in requirements. + +CI installs *only* ``requirements.txt`` (never the pyproject dependency list), +so an import added to the source but not mirrored into ``requirements.txt`` +passes locally — where the dev environment already has it — and only blows up +in CI with ``ModuleNotFoundError``. This static check closes that gap at commit +time: it walks the package's own modules, collects every top-level third-party +import, and verifies each one's distribution is listed in ``requirements.txt``. + +It is intentionally conservative — it only flags *direct* imports (things the +source explicitly ``import``s), never transitive dependencies — because you +should declare what you import rather than lean on someone else's dependency +tree. The pre-push clean-venv run is the definitive backstop for anything this +heuristic misses. + +Usage (defaults auto-detect the single top-level package and repo-root +requirements.txt):: + + python3 scripts/check_requirements_sync.py + python3 scripts/check_requirements_sync.py --package screen_locker \ + --requirements requirements.txt +""" + +from __future__ import annotations + +import argparse +import ast +from pathlib import Path +import re +import sys + +# Import-name -> distribution-name overrides for the rare cases where PEP 503 +# normalisation (lowercase, runs of -/_/. collapsed to '-') is not enough. +# Most packages import under their normalised dist name (crdt_sync -> +# crdt-sync); add an entry here only when they genuinely diverge (the classic +# example being ``import dateutil`` shipped by ``python-dateutil``). +_IMPORT_TO_DIST: dict[str, str] = { + "dateutil": "python-dateutil", + "yaml": "pyyaml", + "kasa": "python-kasa", +} + + +def _normalise(name: str) -> str: + """Return the PEP 503 canonical form of a distribution/import name.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _requirement_names(requirements: Path) -> set[str]: + """Parse distribution names from a requirements.txt file. + + Handles the forms this monorepo uses: ``name``, ``name>=x``, ``name==x``, + ``name[extra]``, and ``name @ git+https://...``. Comments, blank lines and + ``-r``/``-c`` include directives are ignored. + """ + names: set[str] = set() + for raw_line in requirements.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith(("#", "-")): + continue + # The distribution name is the leading token before any version + # specifier, extras bracket, environment marker or ``@`` URL. + token = re.split(r"[\s@<>=!~;\[]", line, maxsplit=1)[0] + if token: + names.add(_normalise(token)) + return names + + +def _iter_package_modules(package_dir: Path) -> list[Path]: + """Return the package's own .py files, excluding any tests directory.""" + return [ + path + for path in sorted(package_dir.rglob("*.py")) + if "tests" not in path.relative_to(package_dir).parts + ] + + +def _top_level_imports(module: Path) -> set[str]: + """Collect top-level module names imported by a single source file. + + ``import a.b`` and ``from a.b import c`` both contribute ``a``. Relative + imports (``from . import x``) are skipped — they are first-party by + definition. + """ + tree = ast.parse(module.read_text(encoding="utf-8"), filename=str(module)) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + imported.add(alias.name.split(".", 1)[0]) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + imported.add(node.module.split(".", 1)[0]) + return imported + + +def _third_party_imports(package_dir: Path, package_name: str) -> set[str]: + """Return third-party top-level imports across the whole package. + + Standard-library modules (via ``sys.stdlib_module_names``) and the package's + own name are filtered out, leaving only external dependencies. + """ + stdlib = sys.stdlib_module_names + external: set[str] = set() + for module in _iter_package_modules(package_dir): + for name in _top_level_imports(module): + if name != package_name and name not in stdlib: + external.add(name) + return external + + +def _distribution_for(import_name: str) -> str: + """Map a top-level import name to its (normalised) distribution name.""" + override = _IMPORT_TO_DIST.get(import_name) + return _normalise(override if override is not None else import_name) + + +def _detect_package(root: Path) -> str: + """Find the single top-level importable package directory under ``root``. + + Raises SystemExit with a helpful message if zero or several candidates + exist, in which case ``--package`` must be given explicitly. + """ + candidates = [ + child.name + for child in sorted(root.iterdir()) + if child.is_dir() + and (child / "__init__.py").is_file() + and child.name != "tests" + ] + if len(candidates) != 1: + sys.exit( + "Could not auto-detect a single package " + f"(found {candidates or 'none'}); pass --package explicitly.", + ) + return candidates[0] + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--package", + help="Top-level package name (auto-detected when omitted).", + ) + parser.add_argument( + "--requirements", + default="requirements.txt", + help="Path to requirements.txt (default: %(default)s).", + ) + parser.add_argument( + "--root", + default=".", + help="Repository root to resolve paths against (default: %(default)s).", + ) + return parser.parse_args(argv) + + +def check(package: str, package_dir: Path, requirements: Path) -> list[str]: + """Return a sorted list of imported dists missing from requirements.txt.""" + declared = _requirement_names(requirements) + missing = { + name + for name in _third_party_imports(package_dir, package) + if _distribution_for(name) not in declared + } + return sorted(missing) + + +def main(argv: list[str] | None = None) -> int: + """Entry point: exit non-zero (with a report) when deps are missing.""" + args = _parse_args(argv) + root = Path(args.root).resolve() + package = args.package or _detect_package(root) + package_dir = root / package + requirements = root / args.requirements + + if not package_dir.is_dir(): + sys.exit(f"Package directory not found: {package_dir}") + if not requirements.is_file(): + sys.exit(f"Requirements file not found: {requirements}") + + missing = check(package, package_dir, requirements) + if missing: + joined = "\n - ".join(missing) + sys.stderr.write( + f"{package}: these third-party imports are not declared in " + f"{requirements.name} (CI installs only that file, so they will " + f"fail with ModuleNotFoundError):\n - {joined}\n", + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_mirror.sh b/scripts/ci_mirror.sh new file mode 100755 index 0000000..66c619d --- /dev/null +++ b/scripts/ci_mirror.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# ============================================================================ +# ci_mirror.sh — run the exact CI gates locally before a push can leave. +# +# Why: the installed pre-commit hook only checks *staged* files, and the local +# dev environment already has every dependency installed. CI does neither — it +# runs `pre-commit run --all-files` and installs a fresh environment from +# requirements.txt. That gap is why green-locally repos went red in CI +# (missing runtime deps, --all-files lint debt, order-dependent test flakes). +# +# This script reproduces both CI workflows on the developer machine: +# 1. a venv built ONLY from requirements.txt (mirrors the Tests workflow's +# `pip install -r requirements.txt`), rebuilt only when requirements.txt +# changes (hash-gated so day-to-day pushes stay fast); +# 2. `pre-commit run --all-files` (mirrors the pre-commit workflow); +# 3. `python -m pytest` inside that clean venv (mirrors Tests). +# +# Wired as the pre-push hook, so a red result blocks the push before CI ever +# sees it. Escape hatch for genuine emergencies: `git push --no-verify`. +# ============================================================================ + +set -euo pipefail + +# Requirements file may be overridden (e.g. testsAndMisc uses meta/…). +REQUIREMENTS_FILE="${REQUIREMENTS_FILE:-requirements.txt}" +readonly REQUIREMENTS_FILE + +ROOT="$(git rev-parse --show-toplevel)" +readonly ROOT +cd "$ROOT" + +readonly VENV_DIR="$ROOT/.ci-mirror-venv" +readonly HASH_FILE="$VENV_DIR/.requirements.sha256" +readonly REQ_PATH="$ROOT/$REQUIREMENTS_FILE" + +log() { printf 'ci-mirror: %s\n' "$1" >&2; } + +fail() { + log "FAILED — $1" + log "CI would be red. Fix the above, or 'git push --no-verify' to override." + exit 1 +} + +require_file() { + if [[ ! -f "$REQ_PATH" ]]; then + fail "requirements file not found: $REQ_PATH" + fi +} + +# Rebuild the venv only when requirements.txt changed since the last build. +ensure_venv() { + local current stored + current="$(sha256sum "$REQ_PATH" | cut -d' ' -f1)" + stored="" + if [[ -f "$HASH_FILE" ]]; then + stored="$(cat "$HASH_FILE")" + fi + + if [[ -x "$VENV_DIR/bin/python" && "$current" == "$stored" ]]; then + log "venv up to date (requirements.txt unchanged)" + return + fi + + log "requirements.txt changed — rebuilding clean venv (mirrors CI install)" + rm -rf "$VENV_DIR" + python3 -m venv "$VENV_DIR" + "$VENV_DIR/bin/python" -m pip install --quiet --upgrade pip \ + || fail "pip self-upgrade in the clean venv" + "$VENV_DIR/bin/python" -m pip install --quiet -r "$REQ_PATH" \ + || fail "pip install -r $REQUIREMENTS_FILE (a dep may be undeclared)" + printf '%s' "$current" > "$HASH_FILE" + log "clean venv ready" +} + +run_precommit_all_files() { + log "pre-commit run --all-files (mirrors the pre-commit workflow)" + pre-commit run --all-files || fail "pre-commit --all-files" +} + +run_pytest_clean_venv() { + log "pytest in the clean venv (mirrors the Tests workflow)" + "$VENV_DIR/bin/python" -m pytest "$@" || fail "pytest (clean requirements.txt venv)" +} + +main() { + require_file + ensure_venv + run_precommit_all_files + run_pytest_clean_venv "$@" + log "all CI gates passed locally — safe to push" +} + +main "$@"