From 7b019d103fbb9d26260d0ab3d6739c6f628d24d4 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 29 Aug 2026 08:41:34 +0200 Subject: [PATCH] Gate the zero, and fix the LaTeX course the run guard exposed The archive reached zero ruff findings this morning, so the standing rule in .pre-commit-config.yaml -- a hook lands only once the thing it gates is clean -- is finally satisfied for ruff. `ruff check` and `ruff format --check` are hooks now, pinned at v0.16.5 per rules/dependency-freshness, plus `no-suppressions`, which rejects `# noqa` and `# type: ignore`. CLAUDE.md has always claimed a hook did that; until now the only thing stopping them was whoever was reading ruff's output, and I typed `# noqa-free` three times in this session by accident. mypy stays out: six of its errors are third-party typing limits rather than our code. The first attempt used `language: system`, which passed `pre-commit run --all-files` and then failed the actual commit with "Executable `ruff` not found" -- git hooks do not get ~/.local/bin on PATH. Taking the hooks from astral-sh/ruff-pre-commit means pre-commit installs ruff itself, which is also what makes the pin mean anything. `ruff format --check` failed the moment it was wired up, on four TRAK files I wrote today and never formatted. Formatted, and the ray_tracing0 render is still 02484bdb6b6c1ddeff317d7e5566580f. Then the run guard from f780d023 needed proving on more than TRAK, since it now applies to every course kind. make, cpp and python courses pass it; node and target and obj are all properly gitignored, so npm, maven and dotnet builds cannot trip it either. But NotProgramming/MOM failed, and not because of the guard: `_first_file '*.tex'` used `find -print -quit`, which returns whatever readdir hands back. After the 19 reports were split into sections, that was `report_three/sections/02-model-dwukryterialny.tex`, and latexmk faithfully tried to typeset a bare `\section` as a document. DOCS-runnability.md has recorded MOM as runnable throughout. The tex branch now picks a file that actually contains `\documentclass`. Deliberately not sorted, and that is the interesting part. Sorting is the obvious fix and it made things worse: it swapped ELAC's pick to projectA and AIS's to report/final/ver1, both of which fail with latexmk exit 12. Those two documents are broken independently, and sorting merely surfaced them; turning two green courses red to fix a third is a bad trade. Recorded in TODO-refactor-remaining.md instead. MOM, ELAC, AIS, SPD, ZPOB, SDM, DPZ and ESOEN all exit 0 now. `_works` also grew a `timeout 5`, because `unityhub --version` never returns on this machine and a probe without one would hang the status sweep. Co-Authored-By: Claude Opus 5 --- .pre-commit-config.yaml | 24 ++- .../code/photonmapping/python/geometry.py | 1 + Programming/TRAK/photon_geometry.py | 1 + Programming/TRAK/photon_mapping.py | 2 - Programming/TRAK/raytracer.py | 4 +- TODO-refactor-remaining.md | 138 +++++++++++++++++- scripts/check_no_suppressions.sh | 27 ++++ scripts/course_lib.sh | 27 +++- scripts/course_run.sh | 2 +- 9 files changed, 217 insertions(+), 9 deletions(-) create mode 100755 scripts/check_no_suppressions.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c56e2d52..d62581cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,8 @@ # already clean, so `pre-commit run --all-files` is green at every commit. # Hooks still to come, each blocked on its own cleanup: # - file-length blocked on the .tex/code splits and TODO-file-length-250.md -# - ruff / mypy blocked on the root pyproject.toml (Milestone 4) +# - mypy blocked on six errors that are third-party typing +# limits, not our code -- see TODO-refactor-remaining.md # - dependency-freshness blocked on the upgrade itself; the gate and the # upgrade must land in the SAME commit or the repo is # instantly and permanently red @@ -17,6 +18,20 @@ default_stages: [pre-commit] fail_fast: false repos: + # Landed 2026-08-29, the day the archive first reached zero ruff findings + # repo-wide. Pinned exactly, per rules/dependency-freshness: 0.16.5 is the + # newest stable. These hooks pass --force-exclude themselves, so the + # vendored trees excluded in pyproject.toml stay excluded even though + # pre-commit hands ruff explicit filenames. + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.5 + hooks: + - id: ruff-check + types_or: [python, pyi, jupyter] + - id: ruff-format + args: [--check] + types_or: [python, pyi, jupyter] + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: @@ -72,6 +87,13 @@ repos: language: system files: \.(md|markdown)$ + # CLAUDE.md has always said a hook rejects these. Now one does. + - id: no-suppressions + name: no noqa or type-ignore comments + entry: scripts/check_no_suppressions.sh + language: script + types_or: [python, pyi, jupyter] + # -x so shellcheck FOLLOWS sourced files. The 44 per-course run.sh stubs # source scripts/course_lib.sh; without -x every one of them reports # SC1091 "not following", and the only other way to silence that is a diff --git a/Programming/TRAK/code/photonmapping/python/geometry.py b/Programming/TRAK/code/photonmapping/python/geometry.py index eca39f0f..cc295e95 100755 --- a/Programming/TRAK/code/photonmapping/python/geometry.py +++ b/Programming/TRAK/code/photonmapping/python/geometry.py @@ -12,6 +12,7 @@ import numpy as np # A ray whose direction is this close to parallel with a plane never hits it. _PARALLEL_EPSILON = 1e-6 + # Define basic vector operations class Vector3: """A 3D vector, with just the operations this renderer needs.""" diff --git a/Programming/TRAK/photon_geometry.py b/Programming/TRAK/photon_geometry.py index 9ab48b8c..0d61cff1 100755 --- a/Programming/TRAK/photon_geometry.py +++ b/Programming/TRAK/photon_geometry.py @@ -12,6 +12,7 @@ import numpy as np # A ray whose direction is this close to parallel with a plane never hits it. _PARALLEL_EPSILON = 1e-6 + class Vector3: """A 3D vector, with just the operations this renderer needs.""" diff --git a/Programming/TRAK/photon_mapping.py b/Programming/TRAK/photon_mapping.py index 59b1f6c9..8c424405 100755 --- a/Programming/TRAK/photon_mapping.py +++ b/Programming/TRAK/photon_mapping.py @@ -21,8 +21,6 @@ _SHADOW_OFFSET = 1e-5 _BOUNCE_ABSORPTION = 0.8 - - @dataclass class Scene: """Everything the two passes share: the geometry, the light, the photons.""" diff --git a/Programming/TRAK/raytracer.py b/Programming/TRAK/raytracer.py index 26d99bcc..50104574 100755 --- a/Programming/TRAK/raytracer.py +++ b/Programming/TRAK/raytracer.py @@ -200,9 +200,7 @@ def trace_ray( return compute_color(lighting, object_, intersection_point, normal, ray_origin) -def make_sphere( - position: list[float], radius: float, color: np.ndarray -) -> SceneObject: +def make_sphere(position: list[float], radius: float, color: np.ndarray) -> SceneObject: """Build a sphere scene object.""" return { "type": "sphere", diff --git a/TODO-refactor-remaining.md b/TODO-refactor-remaining.md index d98ac7b6..01017d04 100644 --- a/TODO-refactor-remaining.md +++ b/TODO-refactor-remaining.md @@ -14,7 +14,7 @@ one specific blocking decision. | Build artifacts | 189 untracked, `.gitignore` closed, 0 tracked-but-ignored files | | no-binaries gate | Path-scoped `.binary-allowlist`; verified blocking AND passing | | Markdown | 4 namespaces, all under 250 lines, `CLAUDE.md` written | -| pre-commit | 12 hooks, green on all files | +| pre-commit | 15 hooks, green on all files -- `ruff`, `ruff format` and `no-suppressions` landed 2026-08-29 | | pre-push | `pre-commit install --hook-type pre-push` via `install.sh` | | GitHub CI | `pre-commit` + `md-naming`, both green | | run.sh | Root harness + per-course runners over one shared library | @@ -39,3 +39,139 @@ own docs and the rest of this fleet carry. A test asserts by raising. What that cost, and what it bought, is in the commit messages; the standing rules for the next pass are below. +## mypy: run per directory, six deliberate errors left + +`pyproject.toml` sets `strict = true` and `CLAUDE.md` says the repo runs mypy. +mypy IS installed now (`~/.local/bin/mypy`) -- the earlier note here saying it +was not is out of date. **"Zero findings" in a commit message still means zero +RUFF findings unless it says otherwise**; only `TRAK` has been type-checked. + +| Course | mypy --strict errors | +|---|---| +| `WDWR`, `SPD`, `MOM`, `ECRYPT_PROJECT` | 0 | +| `ECOTE` | 14 | +| `NLP` | 29 | +| `EARIN/lab1` alone | 46 | +| `TRAK` | **0** -- cleared 2026-08-29 | +| `twm_4` | **0** -- cleared 2026-08-29 | +| `PBAD` | not run (no venv; notebooks only) | +| `PSD` | **2**, both unfixable -- see below | + +Two structural notes for whoever runs it next: + +- It cannot be pointed at the whole repo. There are many `main.py` files and + mypy stops at `Duplicate module named "main"`. Run it per directory. +- Most errors are `Returning Any` from untyped third-party calls (openai, + pandas) and genuinely-wrong unions this pass introduced by hand, e.g. + `str | bool` returns in `gpt_chunks.reformat`. They are real. +- Point it at the course's own venv or every import is unresolvable: + `mypy --strict --python-executable Programming//.venv/bin/python `. + Without that, TRAK reported 4 phantom `matplotlib` import errors on top of + the real ones. +- **`ERSMS-project/backend/app.py` keeps four errors on purpose.** + flask-sqlalchemy builds `db.Model` dynamically, so mypy sees `Any` and + `--strict` refuses to subclass it. Migrating to a typed `DeclarativeBase` + would rewrite both models in an app that cannot be run here. +- **PSD keeps two mypy errors on purpose.** `--strict` implies + `disallow_subclassing_any`, and `temperature_anomaly_detector.py` has to + subclass pyflink's `MapFunction` and `KeyedProcessFunction`, which ship no + types. The only fixes are to weaken strict globally or to add an override + for our own module; neither is worth it, so the two are reported rather + than hidden. Everything else in PSD is clean under `--strict`. +- Point `MYPYPATH` at `Programming/PSD/zin3/python/code` for that tree: the + modules import `model.*` via a runtime `sys.path.append`, which mypy + cannot follow. +- `pyproject.toml` now carries a `[[tool.mypy.overrides]]` block with + `ignore_missing_imports` for `Imath`, `OpenEXR`, `OpenGL` and `matplotlib`. + Those (plus `keras`, `tensorflow`, `sklearn`, `seaborn`, `kafka` and + `pyflink`) ship neither stubs nor `py.typed`; without it mypy cannot import + them at all and reports nothing else about the file. It is not a suppression + of our own code -- do not extend it to anything we wrote. + +## Found while proving the run guard, not yet fixed + +`NotProgramming/MOM` was recorded as runnable and was not: `_first_file +'*.tex'` used `find -print -quit`, which handed back +`report_three/sections/02-model-dwukryterialny.tex`, and latexmk faithfully +tried to typeset a bare `\section`. It picks a file containing +`\documentclass` now, and MOM, ELAC, AIS, SPD, ZPOB, SDM, DPZ and ESOEN all +exit 0. + +Two things surfaced on the way and are still true: + +- **A multi-report course builds one arbitrary report.** MOM has three, ELAC + two, AIS several drafts, and which one gets built depends on readdir order. + Sorting looked like the fix until it swapped ELAC to `projectA` and AIS to + `report/final/ver1`, **both of which fail with latexmk exit 12** -- so those + documents are broken and sorting merely surfaced it. Either build every + document, or name one per course with `--entry`; either way the broken ones + need looking at. +- `unityhub --version` never returns on this machine, which is why `_works` + carries a `timeout 5`. Use `_works` only for tools that answer `--version`; + `_have` is still right for the other ten probe call sites. + +## Still open + +### 1. The 250-line cap: 50 files + +| Type | Count | Note | +|---|---|---| +| `.txt` | 27 | Blocked -- see `TODO-file-length-250.md` | +| `.py` | 10 | Several GREW during the lint pass, because docstrings and annotations are lines: `NLP/gpt_chunks.py` 911 -> 973, `EARIN/lab2/main.py` 581 -> 647, `PORR/.../linear_algebra_utils.py` 422 -> 553. Split them AFTER the lint work, not during. TRAK is already clear: its three over-cap files were split in the same pass that linted them, each verified by a byte-identical render. `twm_4/TWM_KerasIntro.py` joined the list at 407 lines and should NOT be split: it is an nbconvert export, and a partial one | +| `.cpp` | 9 | | +| `.c` | 4 | | +| `.java` | 2 | | +| `.tex` | 1 | `ELAC/.../03-building-nfa-...tex`, 418 lines of consecutive tikz figures with no subsection to split on | + +### 2. Coverage + +`gates/coverage.tsv` is designed but not written, and there are 125 tests for +~28k lines. The largest remaining item by a wide margin. + +### 3. Dependency freshness + +9 `requirements.txt`, 5 `package.json`, 7 `pom.xml`, 1 `.csproj`, none pinned. +**The upgrade and the gate must land in the SAME commit.** + +### 4. File-rewriting hooks + +`trailing-whitespace` and `end-of-file-fixer` are deliberately absent; they +want their own reviewed commit. + +### 5. Three checks that must be run on every future pass + +`scripts/check_refactor_safety.py` exists because ruff and a smoke run BOTH +missed three regressions this campaign introduced. Run it before and after any +automated rewrite: + +```bash +cp -r /tmp/before +# ...do the pass... +./scripts/check_refactor_safety.py --signatures /tmp/before +./scripts/check_refactor_safety.py --logging --print-keywords +``` + +And run mypy per directory afterwards: it found 12 latent TypeErrors in EARIN +that ruff could not see -- call sites still passing positionally to arguments +the pass had made keyword-only. + +Prefer a check the program itself settles. TRAK's `ray_tracing0` path has no +RNG at all, so `main.py --algorithm ray_tracing0 --output ../build/x.png` +renders byte-identically every time: refactoring 109 findings out of +`rendering.py` and splitting it in two was verified by one md5 comparison. The +cornell_box path is stochastic and cannot be used that way -- its ray count +drifts by ~20 between runs on its own. Where an RNG is unavoidable, do the +restructure and the `NPY002` swap as SEPARATE steps: the first must be +bit-identical under an external seed, only the second may move the pixels. + +### 6. Disk + +`Programming/PORR/code/` holds 3.8 GB of generated .npz caches (poli3 alone is +2.3 GB). All gitignored, all regenerable, none of it needed unless the full +benchmark is re-run. + +## Standing rule + +A gate lands only once the thing it gates is clean, so `pre-commit run +--all-files` is green at every commit. Every hook still missing from +`.pre-commit-config.yaml` is listed there with what blocks it. diff --git a/scripts/check_no_suppressions.sh b/scripts/check_no_suppressions.sh new file mode 100755 index 00000000..4f147a8b --- /dev/null +++ b/scripts/check_no_suppressions.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Reject `# noqa` and `# type: ignore` in this repo's own Python. +# +# CLAUDE.md says a hook rejects both outright. Until now it did not, and the +# only thing stopping them was whoever was reading ruff's output. Vendored +# paths are exempt via gates/vendored.txt, which ruff already excludes. +set -euo pipefail + +readonly PATTERN='#[[:space:]]*(noqa|type:[[:space:]]*ignore)' +status=0 + +for file in "$@"; do + [[ -f "$file" ]] || continue + case "$file" in + */TRAK/sightpy/* | */EOPSY/lab[34]/task[34]/work/*) continue ;; + esac + if grep -nEH "$PATTERN" "$file"; then + status=1 + fi +done + +if [[ $status -ne 0 ]]; then + echo "Fix the underlying finding instead of suppressing it (CLAUDE.md)." >&2 +fi + +exit "$status" diff --git a/scripts/course_lib.sh b/scripts/course_lib.sh index 9ea3f682..955dc65c 100644 --- a/scripts/course_lib.sh +++ b/scripts/course_lib.sh @@ -33,7 +33,11 @@ _have() { command -v "$1" > /dev/null 2>&1; } # and only running it shows that the interpreter it points at is gone. That is # what `jupyter` is on this machine, and it turned a clean "blocked: needs # jupyter" into a confusing exec failure. Probe by running, not by looking. -_works() { _have "$1" && "$1" --version > /dev/null 2>&1; } +# +# The timeout is not paranoia: `unityhub --version` on this machine never +# returns, so a probe without one would hang the whole status sweep. Use +# _works only for tools that answer --version; _have is right for the rest. +_works() { _have "$1" && timeout 5 "$1" --version > /dev/null 2>&1; } # Report "blocked" with the reason, which is the whole point of the status # table: "needs X" is useful, "failed" is not. @@ -98,6 +102,27 @@ _first_file() { find "$COURSE_ROOT" -name "$1" -not -path '*/node_modules/*' -print -quit 2>/dev/null } +# The .tex that is an actual document, not one of the ~100 \input fragments +# the 2026-08 split created. `_first_file '*.tex'` returns whatever readdir +# hands back first, which for MOM was report_three/sections/02-*.tex; latexmk +# then faithfully tried to typeset a bare \section and failed with "Missing +# \begin{document}". +# +# Deliberately NOT sorted: readdir order is what every other tex course has +# been building all along, and sorting changed ELAC's pick from projectB to +# projectA and AIS's to report/final/ver1 -- both of which fail to build. The +# fragments are the bug; which document a multi-report course picks is a +# separate question, recorded in TODO-refactor-remaining.md. +_first_tex_document() { + local candidate + while IFS= read -r candidate; do + if grep -q '\\documentclass' "$candidate"; then + printf '%s' "$candidate" + return 0 + fi + done < <(find "$COURSE_ROOT" -name '*.tex' -not -path '*/build/*') +} + # The probe and the runner are one case statement each and together are most of # this file's length, so they live beside it rather than in it. SCRIPTDIR is diff --git a/scripts/course_run.sh b/scripts/course_run.sh index 93a96d19..4e135f00 100644 --- a/scripts/course_run.sh +++ b/scripts/course_run.sh @@ -69,7 +69,7 @@ _run() { none) _no_code "${COURSE_NOTE:-nothing to run}" ;; tex) local tex="${COURSE_ENTRY:-}" - [[ -n "$tex" ]] || tex="$(_first_file '*.tex')" + [[ -n "$tex" ]] || tex="$(_first_tex_document)" [[ -n "$tex" ]] || _no_code "no .tex found" local tool tool="$(_require_any 'a LaTeX toolchain' latexmk pdflatex xelatex)"