diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c56e2d52..d62581cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,8 @@ # already clean, so `pre-commit run --all-files` is green at every commit. # Hooks still to come, each blocked on its own cleanup: # - file-length blocked on the .tex/code splits and TODO-file-length-250.md -# - ruff / mypy blocked on the root pyproject.toml (Milestone 4) +# - mypy blocked on six errors that are third-party typing +# limits, not our code -- see TODO-refactor-remaining.md # - dependency-freshness blocked on the upgrade itself; the gate and the # upgrade must land in the SAME commit or the repo is # instantly and permanently red @@ -17,6 +18,20 @@ default_stages: [pre-commit] fail_fast: false repos: + # Landed 2026-08-29, the day the archive first reached zero ruff findings + # repo-wide. Pinned exactly, per rules/dependency-freshness: 0.16.5 is the + # newest stable. These hooks pass --force-exclude themselves, so the + # vendored trees excluded in pyproject.toml stay excluded even though + # pre-commit hands ruff explicit filenames. + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.5 + hooks: + - id: ruff-check + types_or: [python, pyi, jupyter] + - id: ruff-format + args: [--check] + types_or: [python, pyi, jupyter] + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: @@ -72,6 +87,13 @@ repos: language: system files: \.(md|markdown)$ + # CLAUDE.md has always said a hook rejects these. Now one does. + - id: no-suppressions + name: no noqa or type-ignore comments + entry: scripts/check_no_suppressions.sh + language: script + types_or: [python, pyi, jupyter] + # -x so shellcheck FOLLOWS sourced files. The 44 per-course run.sh stubs # source scripts/course_lib.sh; without -x every one of them reports # SC1091 "not following", and the only other way to silence that is a diff --git a/Programming/TRAK/code/photonmapping/python/geometry.py b/Programming/TRAK/code/photonmapping/python/geometry.py index eca39f0f..cc295e95 100755 --- a/Programming/TRAK/code/photonmapping/python/geometry.py +++ b/Programming/TRAK/code/photonmapping/python/geometry.py @@ -12,6 +12,7 @@ import numpy as np # A ray whose direction is this close to parallel with a plane never hits it. _PARALLEL_EPSILON = 1e-6 + # Define basic vector operations class Vector3: """A 3D vector, with just the operations this renderer needs.""" diff --git a/Programming/TRAK/photon_geometry.py b/Programming/TRAK/photon_geometry.py index 9ab48b8c..0d61cff1 100755 --- a/Programming/TRAK/photon_geometry.py +++ b/Programming/TRAK/photon_geometry.py @@ -12,6 +12,7 @@ import numpy as np # A ray whose direction is this close to parallel with a plane never hits it. _PARALLEL_EPSILON = 1e-6 + class Vector3: """A 3D vector, with just the operations this renderer needs.""" diff --git a/Programming/TRAK/photon_mapping.py b/Programming/TRAK/photon_mapping.py index 59b1f6c9..8c424405 100755 --- a/Programming/TRAK/photon_mapping.py +++ b/Programming/TRAK/photon_mapping.py @@ -21,8 +21,6 @@ _SHADOW_OFFSET = 1e-5 _BOUNCE_ABSORPTION = 0.8 - - @dataclass class Scene: """Everything the two passes share: the geometry, the light, the photons.""" diff --git a/Programming/TRAK/raytracer.py b/Programming/TRAK/raytracer.py index 26d99bcc..50104574 100755 --- a/Programming/TRAK/raytracer.py +++ b/Programming/TRAK/raytracer.py @@ -200,9 +200,7 @@ def trace_ray( return compute_color(lighting, object_, intersection_point, normal, ray_origin) -def make_sphere( - position: list[float], radius: float, color: np.ndarray -) -> SceneObject: +def make_sphere(position: list[float], radius: float, color: np.ndarray) -> SceneObject: """Build a sphere scene object.""" return { "type": "sphere", diff --git a/TODO-refactor-remaining.md b/TODO-refactor-remaining.md index d98ac7b6..01017d04 100644 --- a/TODO-refactor-remaining.md +++ b/TODO-refactor-remaining.md @@ -14,7 +14,7 @@ one specific blocking decision. | Build artifacts | 189 untracked, `.gitignore` closed, 0 tracked-but-ignored files | | no-binaries gate | Path-scoped `.binary-allowlist`; verified blocking AND passing | | Markdown | 4 namespaces, all under 250 lines, `CLAUDE.md` written | -| pre-commit | 12 hooks, green on all files | +| pre-commit | 15 hooks, green on all files -- `ruff`, `ruff format` and `no-suppressions` landed 2026-08-29 | | pre-push | `pre-commit install --hook-type pre-push` via `install.sh` | | GitHub CI | `pre-commit` + `md-naming`, both green | | run.sh | Root harness + per-course runners over one shared library | @@ -39,3 +39,139 @@ own docs and the rest of this fleet carry. A test asserts by raising. What that cost, and what it bought, is in the commit messages; the standing rules for the next pass are below. +## mypy: run per directory, six deliberate errors left + +`pyproject.toml` sets `strict = true` and `CLAUDE.md` says the repo runs mypy. +mypy IS installed now (`~/.local/bin/mypy`) -- the earlier note here saying it +was not is out of date. **"Zero findings" in a commit message still means zero +RUFF findings unless it says otherwise**; only `TRAK` has been type-checked. + +| Course | mypy --strict errors | +|---|---| +| `WDWR`, `SPD`, `MOM`, `ECRYPT_PROJECT` | 0 | +| `ECOTE` | 14 | +| `NLP` | 29 | +| `EARIN/lab1` alone | 46 | +| `TRAK` | **0** -- cleared 2026-08-29 | +| `twm_4` | **0** -- cleared 2026-08-29 | +| `PBAD` | not run (no venv; notebooks only) | +| `PSD` | **2**, both unfixable -- see below | + +Two structural notes for whoever runs it next: + +- It cannot be pointed at the whole repo. There are many `main.py` files and + mypy stops at `Duplicate module named "main"`. Run it per directory. +- Most errors are `Returning Any` from untyped third-party calls (openai, + pandas) and genuinely-wrong unions this pass introduced by hand, e.g. + `str | bool` returns in `gpt_chunks.reformat`. They are real. +- Point it at the course's own venv or every import is unresolvable: + `mypy --strict --python-executable Programming//.venv/bin/python `. + Without that, TRAK reported 4 phantom `matplotlib` import errors on top of + the real ones. +- **`ERSMS-project/backend/app.py` keeps four errors on purpose.** + flask-sqlalchemy builds `db.Model` dynamically, so mypy sees `Any` and + `--strict` refuses to subclass it. Migrating to a typed `DeclarativeBase` + would rewrite both models in an app that cannot be run here. +- **PSD keeps two mypy errors on purpose.** `--strict` implies + `disallow_subclassing_any`, and `temperature_anomaly_detector.py` has to + subclass pyflink's `MapFunction` and `KeyedProcessFunction`, which ship no + types. The only fixes are to weaken strict globally or to add an override + for our own module; neither is worth it, so the two are reported rather + than hidden. Everything else in PSD is clean under `--strict`. +- Point `MYPYPATH` at `Programming/PSD/zin3/python/code` for that tree: the + modules import `model.*` via a runtime `sys.path.append`, which mypy + cannot follow. +- `pyproject.toml` now carries a `[[tool.mypy.overrides]]` block with + `ignore_missing_imports` for `Imath`, `OpenEXR`, `OpenGL` and `matplotlib`. + Those (plus `keras`, `tensorflow`, `sklearn`, `seaborn`, `kafka` and + `pyflink`) ship neither stubs nor `py.typed`; without it mypy cannot import + them at all and reports nothing else about the file. It is not a suppression + of our own code -- do not extend it to anything we wrote. + +## Found while proving the run guard, not yet fixed + +`NotProgramming/MOM` was recorded as runnable and was not: `_first_file +'*.tex'` used `find -print -quit`, which handed back +`report_three/sections/02-model-dwukryterialny.tex`, and latexmk faithfully +tried to typeset a bare `\section`. It picks a file containing +`\documentclass` now, and MOM, ELAC, AIS, SPD, ZPOB, SDM, DPZ and ESOEN all +exit 0. + +Two things surfaced on the way and are still true: + +- **A multi-report course builds one arbitrary report.** MOM has three, ELAC + two, AIS several drafts, and which one gets built depends on readdir order. + Sorting looked like the fix until it swapped ELAC to `projectA` and AIS to + `report/final/ver1`, **both of which fail with latexmk exit 12** -- so those + documents are broken and sorting merely surfaced it. Either build every + document, or name one per course with `--entry`; either way the broken ones + need looking at. +- `unityhub --version` never returns on this machine, which is why `_works` + carries a `timeout 5`. Use `_works` only for tools that answer `--version`; + `_have` is still right for the other ten probe call sites. + +## Still open + +### 1. The 250-line cap: 50 files + +| Type | Count | Note | +|---|---|---| +| `.txt` | 27 | Blocked -- see `TODO-file-length-250.md` | +| `.py` | 10 | Several GREW during the lint pass, because docstrings and annotations are lines: `NLP/gpt_chunks.py` 911 -> 973, `EARIN/lab2/main.py` 581 -> 647, `PORR/.../linear_algebra_utils.py` 422 -> 553. Split them AFTER the lint work, not during. TRAK is already clear: its three over-cap files were split in the same pass that linted them, each verified by a byte-identical render. `twm_4/TWM_KerasIntro.py` joined the list at 407 lines and should NOT be split: it is an nbconvert export, and a partial one | +| `.cpp` | 9 | | +| `.c` | 4 | | +| `.java` | 2 | | +| `.tex` | 1 | `ELAC/.../03-building-nfa-...tex`, 418 lines of consecutive tikz figures with no subsection to split on | + +### 2. Coverage + +`gates/coverage.tsv` is designed but not written, and there are 125 tests for +~28k lines. The largest remaining item by a wide margin. + +### 3. Dependency freshness + +9 `requirements.txt`, 5 `package.json`, 7 `pom.xml`, 1 `.csproj`, none pinned. +**The upgrade and the gate must land in the SAME commit.** + +### 4. File-rewriting hooks + +`trailing-whitespace` and `end-of-file-fixer` are deliberately absent; they +want their own reviewed commit. + +### 5. Three checks that must be run on every future pass + +`scripts/check_refactor_safety.py` exists because ruff and a smoke run BOTH +missed three regressions this campaign introduced. Run it before and after any +automated rewrite: + +```bash +cp -r /tmp/before +# ...do the pass... +./scripts/check_refactor_safety.py --signatures /tmp/before +./scripts/check_refactor_safety.py --logging --print-keywords +``` + +And run mypy per directory afterwards: it found 12 latent TypeErrors in EARIN +that ruff could not see -- call sites still passing positionally to arguments +the pass had made keyword-only. + +Prefer a check the program itself settles. TRAK's `ray_tracing0` path has no +RNG at all, so `main.py --algorithm ray_tracing0 --output ../build/x.png` +renders byte-identically every time: refactoring 109 findings out of +`rendering.py` and splitting it in two was verified by one md5 comparison. The +cornell_box path is stochastic and cannot be used that way -- its ray count +drifts by ~20 between runs on its own. Where an RNG is unavoidable, do the +restructure and the `NPY002` swap as SEPARATE steps: the first must be +bit-identical under an external seed, only the second may move the pixels. + +### 6. Disk + +`Programming/PORR/code/` holds 3.8 GB of generated .npz caches (poli3 alone is +2.3 GB). All gitignored, all regenerable, none of it needed unless the full +benchmark is re-run. + +## Standing rule + +A gate lands only once the thing it gates is clean, so `pre-commit run +--all-files` is green at every commit. Every hook still missing from +`.pre-commit-config.yaml` is listed there with what blocks it. diff --git a/scripts/check_no_suppressions.sh b/scripts/check_no_suppressions.sh new file mode 100755 index 00000000..4f147a8b --- /dev/null +++ b/scripts/check_no_suppressions.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Reject `# noqa` and `# type: ignore` in this repo's own Python. +# +# CLAUDE.md says a hook rejects both outright. Until now it did not, and the +# only thing stopping them was whoever was reading ruff's output. Vendored +# paths are exempt via gates/vendored.txt, which ruff already excludes. +set -euo pipefail + +readonly PATTERN='#[[:space:]]*(noqa|type:[[:space:]]*ignore)' +status=0 + +for file in "$@"; do + [[ -f "$file" ]] || continue + case "$file" in + */TRAK/sightpy/* | */EOPSY/lab[34]/task[34]/work/*) continue ;; + esac + if grep -nEH "$PATTERN" "$file"; then + status=1 + fi +done + +if [[ $status -ne 0 ]]; then + echo "Fix the underlying finding instead of suppressing it (CLAUDE.md)." >&2 +fi + +exit "$status" diff --git a/scripts/course_lib.sh b/scripts/course_lib.sh index 9ea3f682..955dc65c 100644 --- a/scripts/course_lib.sh +++ b/scripts/course_lib.sh @@ -33,7 +33,11 @@ _have() { command -v "$1" > /dev/null 2>&1; } # and only running it shows that the interpreter it points at is gone. That is # what `jupyter` is on this machine, and it turned a clean "blocked: needs # jupyter" into a confusing exec failure. Probe by running, not by looking. -_works() { _have "$1" && "$1" --version > /dev/null 2>&1; } +# +# The timeout is not paranoia: `unityhub --version` on this machine never +# returns, so a probe without one would hang the whole status sweep. Use +# _works only for tools that answer --version; _have is right for the rest. +_works() { _have "$1" && timeout 5 "$1" --version > /dev/null 2>&1; } # Report "blocked" with the reason, which is the whole point of the status # table: "needs X" is useful, "failed" is not. @@ -98,6 +102,27 @@ _first_file() { find "$COURSE_ROOT" -name "$1" -not -path '*/node_modules/*' -print -quit 2>/dev/null } +# The .tex that is an actual document, not one of the ~100 \input fragments +# the 2026-08 split created. `_first_file '*.tex'` returns whatever readdir +# hands back first, which for MOM was report_three/sections/02-*.tex; latexmk +# then faithfully tried to typeset a bare \section and failed with "Missing +# \begin{document}". +# +# Deliberately NOT sorted: readdir order is what every other tex course has +# been building all along, and sorting changed ELAC's pick from projectB to +# projectA and AIS's to report/final/ver1 -- both of which fail to build. The +# fragments are the bug; which document a multi-report course picks is a +# separate question, recorded in TODO-refactor-remaining.md. +_first_tex_document() { + local candidate + while IFS= read -r candidate; do + if grep -q '\\documentclass' "$candidate"; then + printf '%s' "$candidate" + return 0 + fi + done < <(find "$COURSE_ROOT" -name '*.tex' -not -path '*/build/*') +} + # The probe and the runner are one case statement each and together are most of # this file's length, so they live beside it rather than in it. SCRIPTDIR is diff --git a/scripts/course_run.sh b/scripts/course_run.sh index 93a96d19..4e135f00 100644 --- a/scripts/course_run.sh +++ b/scripts/course_run.sh @@ -69,7 +69,7 @@ _run() { none) _no_code "${COURSE_NOTE:-nothing to run}" ;; tex) local tex="${COURSE_ENTRY:-}" - [[ -n "$tex" ]] || tex="$(_first_file '*.tex')" + [[ -n "$tex" ]] || tex="$(_first_tex_document)" [[ -n "$tex" ]] || _no_code "no .tex found" local tool tool="$(_require_any 'a LaTeX toolchain' latexmk pdflatex xelatex)"