From 1d03b508a7d177edf0beba6954af1a8cf81bd750 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 16 Aug 2026 22:59:40 +0200 Subject: [PATCH] Let trace stubs emit values, so traces reach guarded branches A stub that prints nothing makes `size=$(du -sk x)` empty, so every `((size > 0))` is false and the trace walks past every guarded branch. Two such traces compare identical while exercising none of the changed code -- a green check that proves nothing, which is the exact failure mode this harness exists to prevent. --stub now takes `name=output`. Measured on a fixture whose `sudo rm -rf` is guarded by a size test: the silent stub gives `size=[]` / "branch skipped" and never records the rm; `--stub 'du=4096\t/some/dir'` gives `size=[4096]` / "BRANCH TAKEN" and captures it. This weakens the earlier disk_cleanup_check.sh trace, which used a silent du stub: that trace only proved the empty-size path identical. The real run from /tmp is what covered the populated path. Noted in the evidence. Docs also record two limits found while surveying: file-placement installers are not yet traceable (stub cp/chmod/ln and the trace is empty; don't and it rewrites the live system), and baselines for scripts reading relative paths must run from the script's own directory. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY --- docs/shell-split-verification.md | 22 +++++++++- .../trace-harness-stub-values-2026-08-16.json | 43 +++++++++++++++++++ meta/scripts/trace_shell_split.sh | 31 ++++++++++--- 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 docs/superpowers/evidence/trace-harness-stub-values-2026-08-16.json diff --git a/docs/shell-split-verification.md b/docs/shell-split-verification.md index 67921da9..fc2fd151 100644 --- a/docs/shell-split-verification.md +++ b/docs/shell-split-verification.md @@ -73,8 +73,17 @@ exit 1 with the later calls missing from the trace, and the nameref shows up as `circular name reference` in stderr despite exit 0 and correct stdout. A run against a fixture calling `systemctl` left the real unit untouched. -Two things it does not do: +Three things to watch: +- **A silent stub produces an identical, meaningless trace.** This is the + trap. `du` stubbed to print nothing makes `size=$(du -sk x)` empty, so every + `((size > 0))` is false, every guarded branch is skipped, and two such traces + match perfectly while exercising none of the code you changed. Give + value-producing commands a plausible output: `--stub 'du=4096\t/some/dir'`. + Measured on a fixture: the empty stub printed `size=[]` / `branch skipped` + and never reached the guarded `sudo rm -rf`; the valued stub printed + `size=[4096]` / `BRANCH TAKEN` and captured it. Ask of every trace: _did this + actually go down the branch I touched?_ - **Stubs always exit 0.** A script that branches on a real failure status takes a path it would not take live. Hand-write a stub for those and say so in the split's evidence file. @@ -84,6 +93,17 @@ Two things it does not do: target for network and build verbs, then pass them: `--stub git,curl,makepkg`. +**Scripts whose job is placing files** (`install_pacman_wrapper.sh`, +`block_compulsive_opening.sh`) are not yet tractable here: stub `cp`/`chmod`/ +`ln` and the trace is nearly empty, don't stub them and the run rewrites the +live system. Those need a fakeroot-style prefix redirect — a harness change, +not a split. Do not trace one until that exists. + +**Run the baseline from the script's own directory** when it reads relative +paths. `fresh-install/main.sh` reads `aur_packages.txt` and +`pacman_packages.txt` from the cwd; tracing it from elsewhere dies at the first +read and looks exactly like a broken split. + Note it does not need a `main()`, which matters: none of the remaining near-miss shell targets (`install_usage_monitoring.sh`, `install_pacman_wrapper.sh`, `disk_cleanup_check.sh`, `fresh-install/main.sh`) diff --git a/docs/superpowers/evidence/trace-harness-stub-values-2026-08-16.json b/docs/superpowers/evidence/trace-harness-stub-values-2026-08-16.json new file mode 100644 index 00000000..f6a801ee --- /dev/null +++ b/docs/superpowers/evidence/trace-harness-stub-values-2026-08-16.json @@ -0,0 +1,43 @@ +{ + "intent": "Let a stub emit a value so traces exercise guarded branches, closing a hole where two traces compare identical while skipping every branch the split touched.", + "scope": [ + "meta/scripts/trace_shell_split.sh", + "docs/shell-split-verification.md", + "Non-goal: does not make file-placement installers traceable; that needs a prefix redirect" + ], + "changes": [ + "--stub now accepts `name=output`; the generated stub prints that value after recording the call.", + "Generate stubs from a quoted heredoc with the name and value passed via printf %q, so $* and $TRACE_FILE stay literal. Fixes SC2016 by restructuring rather than suppressing.", + "Document the silent-stub trap, the file-placement scripts that remain out of scope, and the need to run baselines from the script's own directory when it reads relative paths." + ], + "verification": [ + { + "command": "trace_shell_split.sh on a fixture whose `sudo rm -rf` is guarded by `((size > 0))`, with --stub du (silent)", + "result": "pass (hole demonstrated)", + "evidence": "stdout 'size=[]' then 'branch skipped'; the guarded `sudo rm -rf` never appears in the trace. Two such traces would match perfectly while exercising nothing." + }, + { + "command": "same fixture with --stub 'du=4096\\t/some/dir'", + "result": "pass", + "evidence": "stdout 'size=[4096]' then 'BRANCH TAKEN'; the trace now records 'sudo rm -rf /some/dir'. The guarded path is reached." + }, + { + "command": "Regressions: the set -e function-tail fixture, the nameref fixture, and the multi-stub git/curl fixture", + "result": "pass", + "evidence": "exit 1 with empty trace; four 'circular name reference' warnings; both network calls captured. Unchanged by the rewrite." + }, + { + "command": "shellcheck + shfmt -d on meta/scripts/trace_shell_split.sh", + "result": "pass", + "evidence": "Both exit 0. SC2016 resolved by the heredoc restructure, not a suppression." + } + ], + "risks": [ + "A stub value is a fixed string, so a script calling the same command with different arguments and expecting different answers gets one answer for all of them. Sufficient for reaching branches; not a simulator.", + "This does not retroactively strengthen the disk_cleanup_check.sh trace, which used a silent du stub. That split's real-run-from-/tmp is what covered the populated path -- the trace alone only proved the empty-size path identical." + ], + "rollback": [ + "Revert the commit; --stub returns to name-only and stubs print nothing.", + "Re-run the four fixtures and confirm the same exit statuses." + ] +} diff --git a/meta/scripts/trace_shell_split.sh b/meta/scripts/trace_shell_split.sh index 0287f6bc..dbd2524a 100755 --- a/meta/scripts/trace_shell_split.sh +++ b/meta/scripts/trace_shell_split.sh @@ -87,15 +87,36 @@ validate_requirements() { # Stubs report success because the point is to reach later code paths, not to # simulate failure. A script whose logic branches on a real exit status needs a # hand-written stub instead -- note that in the split's evidence file. +# +# A stub may carry an output value as `name=text` (from --stub). This matters +# more than it looks: a stub that prints nothing makes every `size=$(du ...)` +# empty, so every `((size > 0))` is false and the trace walks straight past all +# the branches you wanted to compare. Two such traces are identical and +# meaningless. Give value-producing commands a plausible non-zero output. write_stubs() { - local bin_dir="$1" name + local bin_dir="$1" entry name value mkdir -p "$bin_dir" - for name in "${DEFAULT_STUBBED_COMMANDS[@]}" ${EXTRA_STUBS[@]+"${EXTRA_STUBS[@]}"}; do - cat >"$bin_dir/$name" <>"\$TRACE_FILE" + for entry in "${DEFAULT_STUBBED_COMMANDS[@]}" ${EXTRA_STUBS[@]+"${EXTRA_STUBS[@]}"}; do + name="${entry%%=*}" + value="" + if [[ $entry == *=* ]]; then + value="${entry#*=}" + fi + # Quoted heredoc: everything is literal, so $* and $TRACE_FILE reach + # the generated stub instead of expanding here. Only $name and $value + # are interpolated, via the unquoted echo lines around it. + { + echo "#!/usr/bin/env bash" + echo "STUB_NAME=$(printf '%q' "$name")" + echo "STUB_VALUE=$(printf '%q' "$value")" + cat <<'STUB' +printf '%s %s\n' "$STUB_NAME" "$*" >>"$TRACE_FILE" +if [[ -n $STUB_VALUE ]]; then + printf '%s\n' "$STUB_VALUE" +fi exit 0 STUB + } >"$bin_dir/$name" chmod +x "$bin_dir/$name" done }