diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 99613f57..7b2ee9b8 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -12,8 +12,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install bats - run: sudo apt-get update && sudo apt-get install -y bats + # kcov + unshare back shell_coverage_jail.sh, which the jail_args-marked + # lib/tests suites below run under. util-linux (unshare) is preinstalled; + # kcov is not. + - name: Install bats and kcov + run: sudo apt-get update && sudo apt-get install -y bats kcov - name: Run bats suites run: bats linux_configuration/tests/*.bats @@ -69,7 +72,23 @@ jobs: failed=0 for r in "${runners[@]}"; do printf '\n=== %s ===\n' "$r" - if "$r"; then + # A suite that writes to real system paths ships a `jail_args` + # file and REFUSES to run bare; run it through the namespace jail + # with the flags it declares. Empty marker = configuration error. + args_file="${r%/run_all.sh}/jail_args" + if [ -f "$args_file" ]; then + mapfile -t jail_args < <(grep -v '^#\|^$' "$args_file" || true) + if [ "${#jail_args[@]}" -eq 0 ]; then + printf 'FAIL %s (jail_args present but empty)\n' "$r" + failed=$((failed + 1)) + continue + fi + cmd=(bash meta/scripts/shell_coverage_jail.sh \ + --subject "$r" "${jail_args[@]}" -- "") + else + cmd=("$r") + fi + if "${cmd[@]}"; then printf 'PASS %s\n' "$r" else printf 'FAIL %s\n' "$r" diff --git a/docs/superpowers/evidence/jail-aware-libtests-gate-2026-08-22.json b/docs/superpowers/evidence/jail-aware-libtests-gate-2026-08-22.json new file mode 100644 index 00000000..01e8a2ed --- /dev/null +++ b/docs/superpowers/evidence/jail-aware-libtests-gate-2026-08-22.json @@ -0,0 +1,47 @@ +{ + "intent": "Let the six lib/tests suites be gated automatically even when a suite must run inside the coverage jail. Before this, ci_mirror.sh and shell-tests.yml invoked every discovered run_all.sh bare; the features/lib suite refuses to run bare (its REFUSING guards are what stop a test run from rewriting the real /etc), so the push gate blocked all six pending commits and CI would have gone red on the same suite.", + "scope": [ + "meta/scripts/ci_mirror.sh: new run_jailed_suite helper + branch in run_shell_test_gate", + ".github/workflows/shell-tests.yml: same branch in the 'Run lib/tests suites' step; kcov added to the install step", + "linux_configuration/scripts/single_use/features/lib/tests/jail_args: new declarative marker", + "Non-goal: the REFUSING guards are untouched -- they are load-bearing safety, not an obstacle", + "Non-goal: no change to shell_coverage_jail.sh or check_shell_coverage.sh" + ], + "changes": [ + "The jail requirement is now DECLARED BY THE SUITE as a jail_args file beside run_all.sh, holding the --bind/--seed-dir/--seed-file/--timeout flags it needs. Both consumers do one find, then branch on the marker's presence, so adding a jail-requiring suite needs no edit to either runner.", + "Chose data-over-path-list deliberately: the 105 remaining allowlisted libs were scored on root ops and system writes, so most future suites will need the jail too. Hardcoding features/lib in two runners would drift -- the exact failure ci_mirror.sh's own comments say it exists to prevent.", + "Both branches fail closed: a jail_args present but empty is reported as a configuration error, never silently skipped.", + "The flags previously trapped in run_all.sh's header comment are now executable data rather than prose." + ], + "verification": [ + { + "command": "bash meta/scripts/shell_coverage_jail.sh --subject .../features/lib/tests/run_all.sh $(grep -v '^#\\|^$' jail_args) --measure dot_resolver_install.sh --min 100 -- \"\"", + "result": "pass", + "evidence": "dot_resolver_install.sh: 39/39 lines = 100.00% -- the args file reproduces the handoff's independently measured figure, confirming the extracted flags are correct and complete." + }, + { + "command": "for r in ; do timeout 120s \"$r\"; done", + "result": "pass", + "evidence": "PASS on all five: scripts/lib, digital_wellbeing/lib, hosts/lib, periodic_background/lib, single_use/utils/lib. Confirms only features/lib needs the jail, so the bare branch stays correct for the others." + }, + { + "command": "bash -n meta/scripts/ci_mirror.sh && shellcheck meta/scripts/ci_mirror.sh", + "result": "pass", + "evidence": "syntax OK; shellcheck emitted no findings. No suppressions added." + }, + { + "command": "git push origin main (pre-push: prettier + ci-mirror)", + "result": "pass", + "evidence": "ci-mirror previously failed with 'REFUSING: this suite must run under shell_coverage_jail.sh' + 'lib/tests suite failed: .../features/lib/tests/run_all.sh'. After the fix the same gate runs the marked suite through the jail and the push completes." + } + ], + "risks": [ + "CI now needs kcov on ubuntu-latest; it is apt-installable and added to the existing install step, but an apt outage turns the shell-tests job red for an infrastructure reason rather than a code one.", + "A new jail-requiring suite that forgets its jail_args file will be run bare and fail loudly via its own REFUSING guard. That is the intended direction (loud, not silent), but the failure names the guard rather than the missing marker.", + "jail_args is positional shell words, one per line; a flag and its value on the same line would be passed as a single argument and the jail would reject it." + ], + "rollback": [ + "git revert the commit: both runners return to invoking every runner bare, and the jail_args marker disappears.", + "After rollback, re-verify that pushing is blocked again by the features/lib suite -- that block is the symptom this commit removes, so its return confirms a clean revert." + ] +} diff --git a/linux_configuration/scripts/single_use/features/lib/tests/jail_args b/linux_configuration/scripts/single_use/features/lib/tests/jail_args new file mode 100644 index 00000000..d46060e8 --- /dev/null +++ b/linux_configuration/scripts/single_use/features/lib/tests/jail_args @@ -0,0 +1,35 @@ +# Presence of this file next to run_all.sh declares: this suite executes its +# subjects FOR REAL (root writes to /etc, /var, /usr/local/bin) and MUST run +# inside shell_coverage_jail.sh. Running it bare is refused by the suite +# itself; both ci_mirror.sh and shell-tests.yml read this file and wrap the +# runner instead of invoking it directly. +# +# One shell-word per line, blank lines and #-comments ignored. These are the +# jail flags only -- --subject and the trailing cases are supplied by the +# caller. +# +# --bind what the subject WRITES; never what it READS (binding /usr/local/bin +# for a subject that sources siblings from there cut a measurement 75% -> 29%). +# Never --bind /tmp: the jail's own working dir lives there. +--bind +/etc +--bind +/usr/local/bin +--bind +/var +--bind +/root +--seed-dir +/var/www/nextcloud +--seed-dir +/var/lib +--seed-dir +/etc/php/8.2/apache2 +--seed-dir +/etc/php/8.2/mods-available +--seed-dir +/etc/apache2/sites-available +--seed-file +/etc/php/8.2/apache2/php.ini +--timeout +90s diff --git a/meta/scripts/ci_mirror.sh b/meta/scripts/ci_mirror.sh index 3bd8e12b..4703a200 100755 --- a/meta/scripts/ci_mirror.sh +++ b/meta/scripts/ci_mirror.sh @@ -127,6 +127,30 @@ run_gates_in_clean_worktree() { # The Arch-container half of that workflow is deliberately NOT mirrored: it # needs an Arch userland running as root to touch /etc, which is what the # disposable container provides and a developer machine must not. +# A suite that writes to real system paths declares it with a `jail_args` file +# beside its run_all.sh, holding the --bind/--seed-* flags it needs. Such a +# suite REFUSES to run bare -- that guard is what stops a test run from +# rewriting the developer's own /etc -- so invoking it directly fails the gate. +# +# Fails closed: a marker present but empty is a mistake, not a licence to skip. +run_jailed_suite() { + local runner="$1" + local args_file="$CHECKOUT/${runner%/run_all.sh}/jail_args" + + local jail_args=() + while IFS= read -r arg; do + jail_args+=("$arg") + done < <(grep -v '^#\|^$' "$args_file" || true) + + if [[ ${#jail_args[@]} -eq 0 ]]; then + fail "jail_args is present but empty: $args_file" + fi + + (cd "$CHECKOUT" && bash meta/scripts/shell_coverage_jail.sh \ + --subject "$runner" "${jail_args[@]}" -- "" >/dev/null) || + fail "jailed lib/tests suite failed: $runner" +} + run_shell_test_gate() { log "lib/tests suites (mirrors the shell-tests workflow)" local runners=() @@ -141,8 +165,12 @@ run_shell_test_gate() { local runner for runner in "${runners[@]}"; do - (cd "$CHECKOUT" && "$runner" >/dev/null) || - fail "lib/tests suite failed: $runner" + if [[ -f "$CHECKOUT/${runner%/run_all.sh}/jail_args" ]]; then + run_jailed_suite "$runner" + else + (cd "$CHECKOUT" && "$runner" >/dev/null) || + fail "lib/tests suite failed: $runner" + fi done log "side-effect-free shell tests (mirrors the shell-tests workflow)"