From 1351a1460f9fe3b5bb169bfbb207f5b9da2ded2b Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 23 Aug 2026 16:24:17 +0200 Subject: [PATCH] Cache pip and pre-commit envs in CI Measured per-step timings, rather than guessing where the time went: Pre-commit checks Install dependencies 63s (next slowest: 16s) Python tests Install dependencies 49s (pytest itself: 35s) Both workflows re-downloaded ~80 packages from PyPI on every run. The Node workflows already cache npm; the Python ones were the outliers. * cache: pip on actions/setup-python in both, keyed on meta/requirements.txt -- the REAL file, not the root symlink, since the key is hashed from the file at the given path and hashing a symlink is not hashing its target. (That same symlink already cost us a silently-skipped workflow earlier today.) * ~/.cache/pre-commit via actions/cache@v4, keyed on .pre-commit-config.yaml. pre-commit builds a virtualenv per hook repo on first use, which costs more than the pip install itself, and it only needs rebuilding when a hook's repo or rev changes. The first run after this lands is a cache miss and will be no faster; the saving starts from the second. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PJrqKERAo5QUeEHWS5Asgp --- .github/workflows/pre-commit.yml | 12 ++++++ .github/workflows/python-tests.yml | 8 ++++ .../cache-ci-python-deps-2026-08-23.json | 43 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 docs/superpowers/evidence/cache-ci-python-deps-2026-08-23.json diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 6fbe3f3a..ffaa57b1 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -15,6 +15,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" + # See the note in python-tests.yml: caches the pip download cache, + # keyed on the real meta/ path rather than the root symlink. + cache: pip + cache-dependency-path: meta/requirements.txt # zsh backs the zsh-syntax hook. apt does not let a patch release be # selected, so this is pinned only to what the runner image provides; @@ -96,6 +100,14 @@ jobs: fi echo "prettier ${prettier_installed} and shfmt ${shfmt_installed} pinned and on PATH" + # pre-commit builds a virtualenv per hook repo on first use, which is + # slower than the pip install itself. Keyed on the hook definitions, so + # it invalidates exactly when a hook's repo or rev changes. + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-py3.12-${{ hashFiles('.pre-commit-config.yaml') }} + - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 9e161f06..72806e7d 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -43,6 +43,14 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" + # Cache the pip DOWNLOAD cache, keyed on the requirements file, so a + # run that changes nothing about dependencies reuses the wheels + # instead of re-fetching ~80 packages from PyPI. Points at + # meta/requirements.txt, not the root symlink: the cache key is + # hashed from the file at this path, and hashing a symlink is not + # the same as hashing its target. + cache: pip + cache-dependency-path: meta/requirements.txt - name: Install system dependencies run: | set -euo pipefail diff --git a/docs/superpowers/evidence/cache-ci-python-deps-2026-08-23.json b/docs/superpowers/evidence/cache-ci-python-deps-2026-08-23.json new file mode 100644 index 00000000..67c8c3bf --- /dev/null +++ b/docs/superpowers/evidence/cache-ci-python-deps-2026-08-23.json @@ -0,0 +1,43 @@ +{ + "intent": "Stop the two Python CI workflows re-downloading ~80 packages from PyPI on every run, which was the single slowest step in both.", + "scope": [ + ".github/workflows/python-tests.yml", + ".github/workflows/pre-commit.yml", + "Non-goal: pinning dependencies or trimming the requirements list" + ], + "changes": [ + "Added cache: pip with cache-dependency-path: meta/requirements.txt to actions/setup-python in both workflows. The path is the real file, not the root requirements.txt symlink, because the cache key is hashed from the file at the given path.", + "Added an actions/cache@v4 step for ~/.cache/pre-commit in pre-commit.yml, keyed on .pre-commit-config.yaml, so hook virtualenvs are not rebuilt when no hook changed.", + "The Node workflows (poker-stakes, reverse-survivors) already cached npm; the Python ones were the outliers." + ], + "verification": [ + { + "command": "gh api .../actions/runs/32645134828/jobs -- per-step timings for Pre-commit checks", + "result": "pass", + "evidence": "Install dependencies = 63s, the dominant step; every other step 16s or less. This is the step the cache targets." + }, + { + "command": "gh api .../actions/runs/32645075080/jobs -- per-step timings for Python tests", + "result": "pass", + "evidence": "Install dependencies = 49s vs 35s actually running pytest; same dominant step" + }, + { + "command": "python3 -c \"import yaml; yaml.safe_load(...)\" on both workflow files", + "result": "pass", + "evidence": "both parse as valid YAML" + }, + { + "command": "pre-commit run --files .github/workflows/python-tests.yml .github/workflows/pre-commit.yml", + "result": "pass", + "evidence": "no hook reported a failure (prettier, check-yaml included)" + } + ], + "risks": [ + "A stale pip cache could in principle serve an old wheel; mitigated because the key is a hash of the requirements file, so any dependency change invalidates it.", + "The first run after this lands is a cache MISS and will be no faster; the saving appears from the second run onward." + ], + "rollback": [ + "Remove the cache lines from both setup-python blocks and delete the actions/cache step.", + "After rollback, confirm the Install dependencies step returns to ~50-60s." + ] +}