From 2a1bb60d9fd59458ea798a059a28c8ffd13f2c96 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 23 Aug 2026 20:02:59 +0200 Subject: [PATCH] Serialise the heavy ci-mirror gates when memory is tight The push kept failing with "1798 passed" immediately followed by "Killed". The suite was never the problem: it peaks at ~0.29 GiB against its own 2G cap. ci_mirror.sh runs pre-commit --all-files and the pytest gate concurrently, and the pre-commit tree peaks at ~1.75 GiB (mypy and pylint repo-wide, invisible to /usr/bin/time because it measures only the parent). The pair wants ~2 GiB at once; the pytest scope has MemorySwapMax=0, so on a loaded machine it is the one that dies. Concurrency is now conditional on MemAvailable. Above 4 GiB the fast path is unchanged. The probe lives in ci_mirror_mem.sh, following the existing ci_mirror_range/ci_mirror_shell split, which also keeps ci_mirror.sh under the line cap. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PJrqKERAo5QUeEHWS5Asgp --- ...r-serialise-under-pressure-2026-08-23.json | 44 +++++++++++++++++ meta/scripts/ci_mirror.sh | 47 ++++++++++++------- meta/scripts/ci_mirror_mem.sh | 47 +++++++++++++++++++ 3 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 docs/superpowers/evidence/ci-mirror-serialise-under-pressure-2026-08-23.json create mode 100755 meta/scripts/ci_mirror_mem.sh diff --git a/docs/superpowers/evidence/ci-mirror-serialise-under-pressure-2026-08-23.json b/docs/superpowers/evidence/ci-mirror-serialise-under-pressure-2026-08-23.json new file mode 100644 index 00000000..61c02c7b --- /dev/null +++ b/docs/superpowers/evidence/ci-mirror-serialise-under-pressure-2026-08-23.json @@ -0,0 +1,44 @@ +{ + "intent": "Stop the pre-push gate failing with 'tests passed, then Killed' when the machine is under memory pressure, so a push is blocked by the code and not by unrelated load.", + "scope": [ + "meta/scripts/ci_mirror.sh - run the two heavy gates serially when memory is tight", + "meta/scripts/ci_mirror_mem.sh - new lib holding the probe, the threshold and the predicate", + "Non-goal: the 2G pytest cap in pytest_changed_packages.py, which measurement showed is correct and was left alone." + ], + "changes": [ + "Diagnosed the real cause: ci_mirror.sh runs `pre-commit run --all-files` and the pytest gate concurrently with & and wait. Measured tree peaks are ~1.75 GiB for pre-commit (mypy/pylint repo-wide) and ~0.29 GiB for pytest, so the pair wants ~2 GiB at the same instant. The pytest scope runs under MemoryMax=2G with MemorySwapMax=0, so under pressure it is the one the kernel kills - the log reads '1798 passed' then 'Killed', which the gate reports as a test failure.", + "Added available_mem_mib (reads MemAvailable from /proc, no fork) and should_serialise_gates, and made the concurrency conditional on it. Above 4 GiB available the fast parallel path is unchanged.", + "Extracted the probe into ci_mirror_mem.sh following the existing ci_mirror_range.sh / ci_mirror_shell.sh pattern, which also kept ci_mirror.sh under the 250-line cap (245).", + "Exposed the threshold as a predicate rather than a bare constant so SC2034 is resolved by giving the value a real use in its own file, not by a suppression." + ], + "verification": [ + { + "command": "systemd-run --scope -p MemoryMax=2G -p MemorySwapMax=0 /usr/bin/time -v python -m pytest ... (live tree and clean worktree)", + "result": "pass", + "evidence": "1798 passed, Maximum RSS 293928 KB (~0.29 GiB) both times. Proves the suite is nowhere near its own 2G ceiling, so the ceiling was not the problem and did not need raising." + }, + { + "command": "systemd-run --scope -p MemoryMax=8G bash -c 'pre-commit run --all-files; cat .../memory.peak'", + "result": "pass", + "evidence": "1751576576 bytes = ~1.75 GiB across the process tree. /usr/bin/time reported only ~0.29 GiB because it measures the parent, not the forked mypy/pylint children - the cgroup peak is the honest number." + }, + { + "command": "Stubbed available_mem_mib at 1000 and 99999 MiB against should_serialise_gates", + "result": "pass", + "evidence": "1000 -> SERIAL, 99999 -> PARALLEL. Boundary table also checked: 4095 SERIAL, 4096 PARALLEL. Confirms the new branch is reachable and not dead code." + }, + { + "command": "pre-commit run --files meta/scripts/ci_mirror.sh meta/scripts/ci_mirror_mem.sh", + "result": "pass", + "evidence": "No failures. file-length-cap satisfied at 245 lines; SC2034 resolved without a suppression." + } + ], + "risks": [ + "Under 4 GiB available the gate is slower, since the stages no longer overlap. That is the intended trade against a false failure.", + "MemAvailable is a snapshot: memory can be consumed after the check by another process, so this reduces the failure window rather than closing it absolutely. The 2G cap with MemorySwapMax=0 still bounds the damage to a clean kill rather than zram thrashing." + ], + "rollback": [ + "git revert the commit and delete meta/scripts/ci_mirror_mem.sh; the gate returns to unconditional concurrency.", + "After rollback, confirm ci_mirror.sh still sources cleanly (bash -n) and that a push runs the gate." + ] +} diff --git a/meta/scripts/ci_mirror.sh b/meta/scripts/ci_mirror.sh index a4a6b79f..950e6bf5 100755 --- a/meta/scripts/ci_mirror.sh +++ b/meta/scripts/ci_mirror.sh @@ -42,6 +42,8 @@ cd "$ROOT" source "$ROOT/meta/scripts/ci_mirror_range.sh" # shellcheck source=meta/scripts/ci_mirror_shell.sh source "$ROOT/meta/scripts/ci_mirror_shell.sh" +# shellcheck source=meta/scripts/ci_mirror_mem.sh +source "$ROOT/meta/scripts/ci_mirror_mem.sh" # Records the tree hash of the last fully-green run; see tree_cache_*. readonly GREEN_CACHE="$ROOT/.ci-mirror-venv/.last-green-tree" @@ -137,30 +139,41 @@ run_gates_in_clean_worktree() { log "full sweep (no push range, or the gate's own config changed)" fi - # The three stages are independent, so they run concurrently and the gate - # costs the slowest rather than the sum. Output goes to per-stage files and - # is replayed on failure: interleaved live output from three stages is - # unreadable, and the diagnosis is what makes a red gate actionable. + # The stages are independent, so the gate costs the slowest rather than the + # sum when memory allows. Output goes to per-stage files and is replayed on + # failure: interleaved live output is unreadable, and the diagnosis is what + # makes a red gate actionable. local logdir logdir="$(mktemp -d -t ci-mirror-logs-XXXXXX)" LOGDIR="$logdir" # Skip pytest-coverage here: it is a pre-commit-stage hook that would run # in the system env (not our clean venv) and duplicate the pytest run below. - log "pre-commit (mirrors the pre-commit workflow)" - (cd "$CHECKOUT" && SKIP=pytest-coverage pre-commit run "${scope_args[@]}") \ - >"$logdir/precommit.log" 2>&1 & - local pc_pid=$! - - run_python_gate "$paths" >"$logdir/pytest.log" 2>&1 & - local py_pid=$! - - # The shell gate is NOT backgrounded alongside the others when it contains a - # jailed suite: two coverage jails, or a jail racing a busy CPU, produce - # false failures (measured repeatedly). It runs after the parallel pair. + # + # Concurrency is conditional on free memory; ci_mirror_mem.sh has the + # measured peaks and why the pytest scope is the one that gets killed. local pc_rc=0 py_rc=0 - wait "$pc_pid" || pc_rc=$? - wait "$py_pid" || py_rc=$? + if should_serialise_gates; then + log "low memory — running pre-commit and pytest serially" + (cd "$CHECKOUT" && SKIP=pytest-coverage pre-commit run "${scope_args[@]}") \ + >"$logdir/precommit.log" 2>&1 || pc_rc=$? + run_python_gate "$paths" >"$logdir/pytest.log" 2>&1 || py_rc=$? + else + log "pre-commit (mirrors the pre-commit workflow)" + (cd "$CHECKOUT" && SKIP=pytest-coverage pre-commit run "${scope_args[@]}") \ + >"$logdir/precommit.log" 2>&1 & + local pc_pid=$! + + run_python_gate "$paths" >"$logdir/pytest.log" 2>&1 & + local py_pid=$! + + # The shell gate is NOT backgrounded alongside the others when it + # contains a jailed suite: two coverage jails, or a jail racing a busy + # CPU, produce false failures (measured repeatedly). It runs after the + # parallel pair. + wait "$pc_pid" || pc_rc=$? + wait "$py_pid" || py_rc=$? + fi if ((pc_rc != 0)); then cat "$logdir/precommit.log" >&2 diff --git a/meta/scripts/ci_mirror_mem.sh b/meta/scripts/ci_mirror_mem.sh new file mode 100755 index 00000000..0839ea06 --- /dev/null +++ b/meta/scripts/ci_mirror_mem.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +# ============================================================================ +# Memory probe for ci_mirror.sh. +# +# The gate runs `pre-commit run --all-files` and the pytest suite at the same +# time. Measured peaks: the pre-commit tree reaches ~1.75 GiB (mypy and pylint +# over the whole repo) and the pytest scope ~0.29 GiB, so the pair needs +# ~2 GiB simultaneously. The pytest scope runs under MemoryMax=2G with +# MemorySwapMax=0, which means that on a machine already under pressure it is +# the one the kernel kills - surfacing as "1798 passed" immediately followed +# by "Killed" and a failed push that has nothing to do with the code. +# +# Sourced by ci_mirror.sh; not meant to be executed directly. +# ============================================================================ + +# Below this much available memory the two heavy gates run one after the +# other instead of concurrently. 4 GiB is the ~2 GiB they need at the same +# instant plus the same again of headroom, because MemAvailable does not +# account for what they are about to allocate. +readonly SERIAL_BELOW_MIB=4096 + +# MemAvailable in MiB, read straight from /proc rather than parsing `free`. +# Returns a large number when /proc is unreadable so the fast path is kept: +# this is an optimisation guard, not a correctness gate. +available_mem_mib() { + local key value kib="" + while read -r key value _; do + if [[ "$key" == "MemAvailable:" ]]; then + kib="$value" + break + fi + done