diff --git a/docs/superpowers/evidence/session-autopsy-hook-tests-2026-07-25.json b/docs/superpowers/evidence/session-autopsy-hook-tests-2026-07-25.json new file mode 100644 index 00000000..3e76ab51 --- /dev/null +++ b/docs/superpowers/evidence/session-autopsy-hook-tests-2026-07-25.json @@ -0,0 +1,58 @@ +{ + "intent": "Gate the two session_autopsy Claude Code hooks on their observable behaviour. The analyzer package reached 100% branch coverage in a83b5ab, but its only production entry points -- the SessionEnd and SessionStart hook scripts in ~/.claude/hooks -- were untracked, unlinted and untested: the same 'untracked means unlinted' shape a83b5ab's own message describes, one directory over.", + "scope": [ + "linux_configuration/tests/test_session_autopsy_hooks.py (new, only file changed here)", + "Subjects under test: ~/.claude/hooks/session_autopsy_end.sh and session_autopsy_start.sh", + "Non-goal: changing hook behaviour. The hooks are unmodified.", + "Non-goal: re-auditing the 880 python_pkg tests committed in a83b5ab." + ], + "changes": [ + "Added 12 behavioural tests. Both hooks end in an unconditional 'exit 0' by design, so exit status is a constant and asserting it is vacuous; every assertion is on a side effect instead.", + "SessionEnd: a 'setsid' shim on PATH records the launch argv rather than spawning an ingest. Asserts the ingest command, transcript path, --quiet, PYTHONPATH and 'timeout 120' on the happy path; asserts no launch at all for a missing key, empty path, nonexistent file and malformed JSON.", + "SessionStart: asserts exact stdout AND stderr. stderr matters because 'set -u' makes '(( count > 0 ))' on a non-numeric count raise an arithmetic error to stderr while stdout stays silent -- a stdout-only assertion misses a hook injecting bash noise into every session start.", + "Hooks are resolved via AUTOPSY_HOOKS_DIR (default ~/.claude/hooks) so the real deployed artifact is what gets tested, and so the mutation check can run the same suite against a broken copy without touching the live hooks. The suite skips where the hooks are not deployed (CI runners).", + "Test placement is deliberate: a pytest file, not a test_*.sh. Only .py in linux_configuration/tests is picked up by meta/scripts/pytest_changed_packages.py (the local pytest-coverage pre-commit hook); the .sh tests there are gated by the shell-tests CI workflow alone." + ], + "verification": [ + { + "command": "python3 -m pytest linux_configuration/tests/test_session_autopsy_hooks.py", + "result": "pass", + "evidence": "12 passed" + }, + { + "command": "python3 -m pytest (full testpaths, --cov=python_pkg --cov-branch)", + "result": "pass", + "evidence": "977 passed in 6.93s; TOTAL 3899 stmts, 0 miss, 1126 branch, 0 partial, 100%; 'Required test coverage of 100.0% reached'. Up from 965 -- the new file imports nothing from python_pkg, so the 100% gate is untouched." + }, + { + "command": "mutation matrix: gut each hook body; drop the transcript guard; '(( count > 0 ))' -> '>= 0'; drop the numeric regex guard; drop the readability guard; always-announce; and the readability+always-announce compound", + "result": "pass", + "evidence": "12 tests, 12 killed by at least one mutation. Ran per-guard rather than as a single whole-body gutting: gutting alone kills only the 3 positive tests, and would have falsely branded the 9 negative ones vacuous." + }, + { + "command": "shellcheck ~/.claude/hooks/session_autopsy_end.sh ~/.claude/hooks/session_autopsy_start.sh", + "result": "pass", + "evidence": "clean, no findings; hooks unmodified" + }, + { + "command": "pre-commit run --files linux_configuration/tests/test_session_autopsy_hooks.py", + "result": "pass", + "evidence": "all hooks pass. One real mypy finding was fixed rather than suppressed: pytest.fail is not seen as NoReturn here, so _await_launch raised 'Missing return statement'; it now raises AssertionError. No noqa or type: ignore added." + }, + { + "command": "live end-to-end: pipe a real SessionEnd payload into the deployed hook, first under an isolated HOME, then against the real store", + "result": "pass", + "evidence": "Isolated run ingested a real 9KB transcript into a scratch store with an empty hook.log. Live run: exit 0; sessions.jsonl rows 264 -> 264 and unreviewed_count 178 -> 178, with the session present exactly once, confirming store.py:117 upsert_records replaces by session_id rather than appending. The AttributeError tracebacks in hook.log are stale (last written 15:56, before a83b5ab); nothing was appended by this run." + } + ], + "risks": [ + "The gate is local-pre-commit only. CI skips the suite (no ~/.claude on the runner), and pytest_changed_packages.py is types: [python], so it fires when this test file is in a testsAndMisc commit -- editing a hook inside ~/.claude, which has no .pre-commit-config.yaml, triggers nothing. Closing that means giving ~/.claude its own pre-commit config: a separate task.", + "The 'no launch' assertions sleep a 1s settle window because the hook detaches with 'setsid ... &'. Asserting absence the instant the parent exits would pass even against a hook whose guard had been deleted. Cost: about 4s of suite time, spent deliberately to keep those four tests non-vacuous.", + "test_unreadable_state relies on running as non-root; root bypasses the readability check, so it is skipped there. It must not be moved into the root arch-tests CI job.", + "Finding, not acted on: the start hook's '[[ ! -r \"$STATE\" ]]' guard is provably redundant -- deleting it kills zero tests, because jq's stderr is suppressed and an empty count already fails the arithmetic. Harmless defence in depth; left alone because hook behaviour changes were out of scope." + ], + "rollback": [ + "git revert this commit, or delete linux_configuration/tests/test_session_autopsy_hooks.py -- no production code is touched, so reverting removes only test coverage.", + "After rollback, confirm 'python3 -m pytest' still reports 965 passed at 100% coverage and that both hooks remain executable and shellcheck-clean." + ] +} diff --git a/linux_configuration/tests/test_session_autopsy_hooks.py b/linux_configuration/tests/test_session_autopsy_hooks.py new file mode 100644 index 00000000..2976cee0 --- /dev/null +++ b/linux_configuration/tests/test_session_autopsy_hooks.py @@ -0,0 +1,223 @@ +"""Behavioural tests for the two session_autopsy Claude Code hooks. + +Both hooks end in an unconditional ``exit 0`` by design — an autopsy failure must +never surface into the session lifecycle — so exit status is a constant and +asserting it proves nothing. Every assertion here is on an observable side +effect instead: what the SessionEnd hook hands to ``setsid``, and what the +SessionStart hook writes to stdout. + +The hooks under test are the real deployed artifacts in ``~/.claude/hooks``. +``AUTOPSY_HOOKS_DIR`` overrides that, which is what lets the mutation check run +the same suite against a deliberately broken copy without touching the live +hooks. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import time + +import pytest + +_HOOKS_DIR = Path( + os.environ.get("AUTOPSY_HOOKS_DIR", Path.home() / ".claude" / "hooks") +) +_END_HOOK = _HOOKS_DIR / "session_autopsy_end.sh" +_START_HOOK = _HOOKS_DIR / "session_autopsy_start.sh" + +# The hooks live in ~/.claude, which does not exist on a CI runner. +pytestmark = pytest.mark.skipif( + not (_END_HOOK.is_file() and _START_HOOK.is_file()), + reason=f"session_autopsy hooks are not deployed under {_HOOKS_DIR}", +) + +_LAUNCH_WAIT_SECONDS = 5.0 +_POLL_SECONDS = 0.02 +_RUN_TIMEOUT_SECONDS = 30 + +# The hook detaches its ingest with ``setsid ... &``, so the parent can exit +# before the shim has recorded anything. Asserting "no launch" the instant the +# parent exits would therefore pass even against a hook whose guard had been +# deleted — vacuously. This window gives a launch that WAS made time to show up. +_SETTLE_SECONDS = 1.0 + + +def _isolated_env(home: Path, *extra_path: Path) -> dict[str, str]: + """Build an environment whose ``HOME`` — and so every hook path — is ``home``.""" + env = dict(os.environ) + env["HOME"] = str(home) + if extra_path: + joined = os.pathsep.join(str(entry) for entry in extra_path) + env["PATH"] = f"{joined}{os.pathsep}{env['PATH']}" + return env + + +def _write_setsid_shim(bin_dir: Path, record: Path) -> None: + """Install a ``setsid`` that records its argv instead of launching an ingest.""" + bin_dir.mkdir(parents=True, exist_ok=True) + shim = bin_dir / "setsid" + shim.write_text( + f'#!/bin/bash\nprintf "%s\\n" "$*" >> {record}\nexit 0\n', + encoding="utf-8", + ) + shim.chmod(0o755) + + +def _run_end_hook(tmp_path: Path, payload: str) -> tuple[Path, Path]: + """Run the SessionEnd hook on ``payload``; return its fake HOME and argv record. + + The record is written by the ``setsid`` shim, so its existence means the hook + decided to launch an ingest, and its absence means a guard stopped it. + """ + home = tmp_path / "home" + home.mkdir(exist_ok=True) + record = tmp_path / "setsid.argv" + _write_setsid_shim(tmp_path / "bin", record) + subprocess.run( + [str(_END_HOOK)], + input=payload, + env=_isolated_env(home, tmp_path / "bin"), + capture_output=True, + text=True, + timeout=_RUN_TIMEOUT_SECONDS, + check=False, + ) + return home, record + + +def _await_launch(record: Path) -> str: + """Return the recorded argv, waiting for the backgrounded ``setsid &`` to land.""" + deadline = time.monotonic() + _LAUNCH_WAIT_SECONDS + while time.monotonic() < deadline: + if record.is_file(): + return record.read_text(encoding="utf-8") + time.sleep(_POLL_SECONDS) + msg = f"no ingest was launched within {_LAUNCH_WAIT_SECONDS}s" + raise AssertionError(msg) + + +def _assert_no_launch(record: Path) -> None: + """Assert no ingest was launched, allowing for the hook's detached ``&``.""" + time.sleep(_SETTLE_SECONDS) + assert not record.exists() + + +def _run_start_hook( + tmp_path: Path, state: str | None, *, mode: int | None = None +) -> tuple[str, str]: + """Run the SessionStart hook against ``state``; return ``(stdout, stderr)``. + + stderr matters as much as stdout here: this hook runs at every session start, + and bash noise (an arithmetic error on a non-numeric count, say) would be + injected into the session just as surely as an intentional banner. + """ + home = tmp_path / "home" + autopsy = home / ".claude" / "autopsy" + autopsy.mkdir(parents=True, exist_ok=True) + if state is not None: + state_file = autopsy / "state.json" + state_file.write_text(state, encoding="utf-8") + if mode is not None: + state_file.chmod(mode) + result = subprocess.run( + [str(_START_HOOK)], + env=_isolated_env(home), + capture_output=True, + text=True, + timeout=_RUN_TIMEOUT_SECONDS, + check=False, + ) + return result.stdout, result.stderr + + +class TestSessionEndHookLaunchesIngest: + """A usable transcript is handed to the analyzer, detached.""" + + def test_launches_ingest_for_the_transcript(self, tmp_path: Path) -> None: + transcript = tmp_path / "session.jsonl" + transcript.write_text('{"type":"user"}\n', encoding="utf-8") + home, record = _run_end_hook( + tmp_path, + json.dumps({"transcript_path": str(transcript)}), + ) + argv = _await_launch(record) + assert "python_pkg.session_autopsy" in argv + assert "ingest" in argv + assert str(transcript) in argv + assert "--quiet" in argv + assert f"PYTHONPATH={home}/testsAndMisc" in argv + assert "timeout 120" in argv + + def test_creates_the_autopsy_home(self, tmp_path: Path) -> None: + transcript = tmp_path / "session.jsonl" + transcript.write_text("{}\n", encoding="utf-8") + home, record = _run_end_hook( + tmp_path, + json.dumps({"transcript_path": str(transcript)}), + ) + _await_launch(record) + assert (home / ".claude" / "autopsy").is_dir() + + +class TestSessionEndHookDeclinesToLaunch: + """Nothing is launched without a transcript that exists.""" + + def test_missing_key(self, tmp_path: Path) -> None: + _, record = _run_end_hook(tmp_path, "{}") + _assert_no_launch(record) + + def test_empty_path(self, tmp_path: Path) -> None: + _, record = _run_end_hook(tmp_path, json.dumps({"transcript_path": ""})) + _assert_no_launch(record) + + def test_nonexistent_path(self, tmp_path: Path) -> None: + payload = json.dumps({"transcript_path": str(tmp_path / "gone.jsonl")}) + _, record = _run_end_hook(tmp_path, payload) + _assert_no_launch(record) + + def test_malformed_payload(self, tmp_path: Path) -> None: + _, record = _run_end_hook(tmp_path, "not json at all") + _assert_no_launch(record) + + +class TestSessionStartHookAnnounces: + """The nudge is printed only when there is something to review.""" + + def test_announces_unreviewed_candidates(self, tmp_path: Path) -> None: + stdout, stderr = _run_start_hook(tmp_path, json.dumps({"unreviewed_count": 7})) + assert stderr == "" + assert stdout.count("\n") == 1 + assert "7 unreviewed automation candidates" in stdout + assert "REPORT.md" in stdout + assert "/compile-candidate" in stdout + + +class TestSessionStartHookStaysSilent: + """Every path that is not a positive count costs zero context tokens. + + Silence means BOTH streams: a bash error on stderr reaches the session too. + """ + + def test_no_state_file(self, tmp_path: Path) -> None: + assert _run_start_hook(tmp_path, None) == ("", "") + + def test_zero_count(self, tmp_path: Path) -> None: + assert _run_start_hook(tmp_path, json.dumps({"unreviewed_count": 0})) == ( + "", + "", + ) + + def test_non_numeric_count(self, tmp_path: Path) -> None: + state = json.dumps({"unreviewed_count": "abc"}) + assert _run_start_hook(tmp_path, state) == ("", "") + + def test_malformed_state(self, tmp_path: Path) -> None: + assert _run_start_hook(tmp_path, "{not json") == ("", "") + + @pytest.mark.skipif(os.geteuid() == 0, reason="root bypasses the readability check") + def test_unreadable_state(self, tmp_path: Path) -> None: + state = json.dumps({"unreviewed_count": 9}) + assert _run_start_hook(tmp_path, state, mode=0o000) == ("", "")