From b7523d1691bcff301c8594a110841529af3e643a Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 22 Aug 2026 11:40:16 +0200 Subject: [PATCH] Bound every coverage-jail case with a timeout and closed stdin Investigated whether this session caused the 11:33 hard freeze. It did not, and the evidence is recorded in the evidence artifact: - No OOM kill, no oom_reaper, no hung-task or soft-lockup message, and no kernel message AT ALL after 11:20. The log stops mid-line at 11:33:15 -- a hard hang, not a process the kernel was fighting. - Last commit 11:30:32; last jail trace 11:11:57, 21 minutes earlier. The session was idle when the machine died. - Steam wrote a crash minidump at 11:31:10 and at-spi2 disabled an unresponsive app at 11:32:06, on an RTX 3090. Correlation only -- a hard hang leaves no log to prove cause. - Nothing survived the reboot: no stray unshare/kcov, no leftover tmpdirs, no leaked mounts, and the live /etc and /usr/local/bin files keep their original dates. Hardened anyway, because the jail had one genuinely unbounded behaviour: a case could run forever. Two hangs yesterday came from a `crontab -` fallback inheriting the caller's stdin. Each case now runs under `timeout --kill-after=10s` (default 120s, --timeout to override) with stdin from /dev/null, and a timed-out case WARNS by name instead of being swallowed -- a silent timeout looks identical to an unreachable line in the report. Verified: a `sleep 600` subject now ends in 5s with --timeout 5s. dot_resolver_install.sh still measures 39/39 = 100.00%. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DbD4nco3EgL1UFoZKmKsLE --- .../jail-case-timeout-2026-08-22.json | 53 +++++++++++++++++++ meta/scripts/shell_coverage_jail.sh | 27 ++++++++-- 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 docs/superpowers/evidence/jail-case-timeout-2026-08-22.json diff --git a/docs/superpowers/evidence/jail-case-timeout-2026-08-22.json b/docs/superpowers/evidence/jail-case-timeout-2026-08-22.json new file mode 100644 index 00000000..6821c1da --- /dev/null +++ b/docs/superpowers/evidence/jail-case-timeout-2026-08-22.json @@ -0,0 +1,53 @@ +{ + "intent": "Investigate whether this session's work caused the 2026-08-22 11:33 hard freeze, and bound the one unbounded thing the coverage jail could do regardless of the verdict.", + "scope": [ + "meta/scripts/shell_coverage_jail.sh: per-case timeout and stdin closed", + "Non-goal: changing anything else about the jail; the freeze investigation cleared it" + ], + "changes": [ + "Each kcov case now runs under `timeout --kill-after=10s` with a default 120s ceiling, overridable with --timeout.", + "Each case runs with stdin redirected from /dev/null, so a subject that reads stdin cannot block on the caller's terminal.", + "A timed-out case prints a warning naming the case, rather than being swallowed by the existing `|| true`." + ], + "verification": [ + { + "command": "journalctl --list-boots; journalctl -b -1 for OOM, hung tasks, lockups and late kernel messages", + "result": "pass - this session's work is not implicated", + "evidence": "Boot -1 ended 11:33:15 with the log cut mid-line. No OOM kill, no oom_reaper, no hung-task or soft-lockup message anywhere, and NO kernel message at all after 11:20 -- a hard hang signature rather than a process the kernel was fighting." + }, + { + "command": "correlate this session's activity with the final minutes of boot -1", + "result": "pass - the session was idle when the freeze happened", + "evidence": "Last commit 11:30:32; last system-level trace from a jail run at 11:11:57 (a failed sudo from an early attempt), over 21 minutes before the freeze. Meanwhile Steam wrote a crash minidump at 11:31:10 and at-spi2 disabled an unresponsive app (pid 4848) at 11:32:06, on an RTX 3090 with the proprietary driver." + }, + { + "command": "ps for unshare/kcov; ls /tmp/tmp.*; mount | grep /tmp/tmp.; stat the live system files", + "result": "pass - no residue survived the reboot", + "evidence": "No stray processes, no leftover jail tmpdirs, zero leaked mounts. /etc/modules-load.d/night-lockdown-i2c.conf still dated 2026-07-23 and /usr/local/bin/sync-dot-cert.sh still dated 2026-08-11, both untouched." + }, + { + "command": "bash meta/scripts/shell_coverage_jail.sh ... --measure dot_resolver_install.sh --min 100", + "result": "pass - no regression from the timeout change", + "evidence": "dot_resolver_install.sh: 39/39 lines = 100.00%, both before and after." + }, + { + "command": "run a subject whose body is `sleep 600` with --timeout 5s", + "result": "pass - the bound fires", + "evidence": "'warn: case timed out after 5s' and the whole run finished in 5 seconds instead of 600." + }, + { + "command": "shellcheck meta/scripts/shell_coverage_jail.sh; check_file_length.sh", + "result": "pass", + "evidence": "Clean, no suppressions; the file is exactly at the 250-line cap." + } + ], + "risks": [ + "The freeze cause is NOT positively identified. A hard hang leaves no log by construction, so the Steam minidump two minutes prior is a correlation, not a proven cause. What is established is the negative: nothing from this session was running at the time, and no OOM occurred.", + "The jail still runs real installers as uid 0 inside a namespace. That is the most invasive thing this tooling does, and while containment was verified by canary, escape assertion and mtime checks, it remains the part with any plausible blast radius.", + "The 120s default is a guess, not a measured figure. A legitimately slow suite could now be cut off; the warning names the case so that failure is visible rather than silent." + ], + "rollback": [ + "git revert this commit; the jail returns to unbounded per-case execution.", + "After rollback, confirm dot_resolver_install.sh still measures 39/39 through the runner." + ] +} diff --git a/meta/scripts/shell_coverage_jail.sh b/meta/scripts/shell_coverage_jail.sh index 7d471c45..b11d0a43 100755 --- a/meta/scripts/shell_coverage_jail.sh +++ b/meta/scripts/shell_coverage_jail.sh @@ -43,6 +43,9 @@ MEASURE="" SEED_DIRS=() SEED_FILES=() MIN_PERCENT=100 +# Per-case wall-clock ceiling. A hung case cannot be distinguished from a slow +# one without a bound, and an unbounded run blocks a hook indefinitely. +CASE_TIMEOUT="120s" BIND_PATHS=() EXTRA_SHIMS=() CASES=() @@ -64,6 +67,7 @@ Usage: $SCRIPT_NAME --subject [options] -- [...] script writing /etc/php/8.2/apache2/ finds nothing to mirror on an Arch box. --seed-file create an empty file at inside the jail + --timeout per-case wall-clock ceiling (default 120s) --min fail below this (default 100) -- ... one invocation each; "" means "no arguments" @@ -101,6 +105,10 @@ while [[ $# -gt 0 ]]; do SEED_FILES+=("$2") shift 2 ;; + --timeout) + CASE_TIMEOUT="$2" + shift 2 + ;; --min) MIN_PERCENT="$2" shift 2 @@ -183,6 +191,7 @@ jail="$1" subject_dir="$2" subject_base="$3" measure_base="$4" +case_timeout="$5" . "$jail/mounts.sh" export PATH="$jail/bin:/usr/bin:/bin" export HOME="$jail/home" @@ -192,9 +201,20 @@ export SUDO_USER="$USER" cd "$subject_dir" || exit 1 while IFS= read -r line; do read -r -a case_args <<<"$line" - kcov --include-pattern="$measure_base" "$jail/cov" \ + # Bounded, and with stdin closed. Both matter: a subject that reads stdin + # blocks forever on the caller's terminal (a `crontab -` fallback did + # exactly this and hung a run for the full two minutes with no output), + # and a subject that spins never yields at all. A timed-out case is + # reported rather than swallowed, because a silent timeout looks + # identical to an unreachable line in the coverage report. + timeout --kill-after=10s "$case_timeout" \ + kcov --include-pattern="$measure_base" "$jail/cov" \ "./$subject_base" ${case_args[@]+"${case_args[@]}"} \ - >/dev/null 2>&1 || true + >/dev/null 2>&1 &2 + fi done <"$jail/cases" INNER chmod +x "$JAIL/run_cases.sh" @@ -212,7 +232,8 @@ INNER } >>"$JAIL/mounts.sh" unshare --user --map-root-user --mount --fork \ - "$JAIL/run_cases.sh" "$JAIL" "$subject_dir" "$subject_base" "$measure_base" + "$JAIL/run_cases.sh" "$JAIL" "$subject_dir" "$subject_base" "$measure_base" \ + "$CASE_TIMEOUT" local tree_after tree_after="$(cd "$REPO_ROOT" && git status --porcelain)"