diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3c39142..87715750 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -252,6 +252,16 @@ repos: language: system pass_filenames: false files: ^(python_pkg/.*\.py|requirements\.txt)$ + - id: session-autopsy-hook-mutations + name: session_autopsy hook tests are non-vacuous (mutation) + entry: python3 meta/scripts/mutate_session_autopsy_hooks.py + language: system + pass_filenames: false + require_serial: true + # Scoped to the two files that can invalidate the proof: the suite + # itself and the mutation definitions. Takes ~40s, so it must not run + # on unrelated commits. Skips itself where ~/.claude is absent (CI). + files: ^(linux_configuration/tests/test_session_autopsy_hooks\.py|meta/scripts/mutate_session_autopsy_hooks\.py)$ - id: ci-mirror name: CI mirror (clean-venv install + --all-files + changed-pkg pytest) entry: bash meta/scripts/ci_mirror.sh diff --git a/docs/superpowers/evidence/session-autopsy-hook-mutation-gate-2026-07-25.json b/docs/superpowers/evidence/session-autopsy-hook-mutation-gate-2026-07-25.json new file mode 100644 index 00000000..822d7e1c --- /dev/null +++ b/docs/superpowers/evidence/session-autopsy-hook-mutation-gate-2026-07-25.json @@ -0,0 +1,47 @@ +{ + "intent": "Make the non-vacuity of the session_autopsy hook tests re-verifiable and enforced, instead of a claim in a commit message. 63a5124 asserted '12 tests, 12 killed by at least one mutation', but the harness that proved it lived in a session scratchpad and would have vanished with the session -- leaving an unfalsifiable claim about tests whose whole purpose is to not be vacuous.", + "scope": [ + "meta/scripts/mutate_session_autopsy_hooks.py (new)", + ".pre-commit-config.yaml (one new local hook)", + "Non-goal: changing the hooks or the tests. Both are unmodified by this commit." + ], + "changes": [ + "Promoted the scratchpad harness to a repo tool: 8 named mutations, each breaking ONE guard in a copy of the hooks, run against the suite through AUTOPSY_HOOKS_DIR. Fails if any test survives every mutation.", + "Per-guard rather than whole-body mutation, because gutting a hook body kills only the 3 positive tests -- a gutted hook emits neither a launch nor output, which is exactly what the 9 negative tests expect. Whole-body gutting alone would falsely brand them vacuous.", + "Added a staleness guard: if a mutation leaves the hook text unchanged it raises rather than silently reporting 'killed 0'. A behaviour-preserving edit to a hook (renaming a variable, say) would otherwise turn a mutation into a no-op and quietly weaken the proof.", + "Exits 0 when the hooks are not deployed, so a CI runner with no ~/.claude skips rather than fails.", + "Wired as a pre-commit hook scoped by files: regex to the suite and the mutation definitions -- the only two files that can invalidate the proof. It takes ~40s, so it must not run on unrelated commits.", + "The tool is a gate, not a report: it returns non-zero on survivors, per the 'self-checks must gate, not warn' rule." + ], + "verification": [ + { + "command": "python3 meta/scripts/mutate_session_autopsy_hooks.py", + "result": "pass", + "evidence": "8 mutations; 'PASS: all 12 tests killed by at least one mutation'. Per-mutation kills: gut-end 2, gut-start 1, end-no-transcript-guard 4, start-count-ge-zero 1, start-no-numeric-guard 1, start-no-readable-guard 0, start-always-announce 3, start-unguarded-announce 5." + }, + { + "command": "AUTOPSY_HOOKS_DIR=/nonexistent/hooks python3 meta/scripts/mutate_session_autopsy_hooks.py", + "result": "pass", + "evidence": "'session_autopsy hooks are not deployed under /nonexistent/hooks -- skipping', exit 0. This is the CI path." + }, + { + "command": "staleness guard: copy the hooks, sed 's/\\bcount\\b/n/g' the start hook (behaviour-identical rename), run against the copy", + "result": "pass", + "evidence": "The drifted hook still printed the correct banner, confirming the rename preserved behaviour; the tool then failed with \"mutation 'start-count-ge-zero' no longer applies to session_autopsy_start.sh -- the hook text changed, so this mutation tests nothing.\" Text drift is detected without a behaviour break." + }, + { + "command": "pre-commit run --files meta/scripts/mutate_session_autopsy_hooks.py .pre-commit-config.yaml linux_configuration/tests/test_session_autopsy_hooks.py", + "result": "pass", + "evidence": "All hooks pass, including the new 'session_autopsy hook tests are non-vacuous (mutation)'. Two real lint findings were fixed rather than suppressed: an import inside a function (ruff PLC0415 / pylint C0415) moved to module scope, and unsorted imports. No noqa or type: ignore added." + } + ], + "risks": [ + "The gate adds ~40s to any commit touching either of the two scoped files. Deliberate: it is the only thing that can catch the tests silently decaying into exit-status assertions.", + "Mutations are string transforms over the hook text, so they are coupled to that text by construction. The staleness guard converts that coupling from a silent weakening into a loud failure, but it does mean editing a hook can require updating a mutation.", + "start-no-readable-guard kills nothing on its own; this is a recorded finding, not a gap. The start hook's readability guard is redundant -- jq's stderr is suppressed and an empty count already fails the arithmetic -- so only the compound start-unguarded-announce mutation reaches those cases." + ], + "rollback": [ + "Remove the session-autopsy-hook-mutations hook from .pre-commit-config.yaml and delete meta/scripts/mutate_session_autopsy_hooks.py. No production code or test behaviour is affected; the 12 tests keep running under the ordinary pytest gate.", + "After rollback, confirm 'pre-commit run --files linux_configuration/tests/test_session_autopsy_hooks.py' still passes and the suite still reports 12 passed." + ] +} diff --git a/meta/scripts/mutate_session_autopsy_hooks.py b/meta/scripts/mutate_session_autopsy_hooks.py new file mode 100755 index 00000000..b8c31b10 --- /dev/null +++ b/meta/scripts/mutate_session_autopsy_hooks.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Prove the session_autopsy hook tests are non-vacuous, by mutation. + +Both hooks in ``~/.claude/hooks`` end in an unconditional ``exit 0`` by design, +so a test that asserts exit status passes against a hook whose body has been +deleted. ``linux_configuration/tests/test_session_autopsy_hooks.py`` therefore +asserts side effects instead -- but "asserts a side effect" is a claim, and this +script is what checks it. + +Each mutation below breaks ONE guard in a copy of the hooks, points the suite at +that copy through ``AUTOPSY_HOOKS_DIR`` and records which tests died. A test that +no mutation kills is asserting nothing worth asserting. The deployed hooks are +never modified. + +Mutating per guard matters: gutting a whole hook body kills only the positive +tests, because a gutted hook emits neither a launch nor any output -- exactly +what the negative tests already expect. Run that way, nine of the twelve tests +look vacuous when they are not. + +Used as a pre-commit hook entry point and runnable by hand. Exits 0 when the +hooks are not deployed (a CI runner has no ``~/.claude``), since there is then +nothing to mutate. +""" + +from __future__ import annotations + +import ast +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +from typing import TYPE_CHECKING, NamedTuple + +if TYPE_CHECKING: + from collections.abc import Callable + +_TEST_FILE = "linux_configuration/tests/test_session_autopsy_hooks.py" +_END_HOOK = "session_autopsy_end.sh" +_START_HOOK = "session_autopsy_start.sh" +_GUTTED = "#!/bin/bash\nexit 0\n" +_OUTCOME = re.compile(r"::(test_\w+)\s+(PASSED|FAILED|ERROR|SKIPPED)") + + +def _emit(message: str) -> None: + """Write ``message`` and a newline to stdout (``print`` is banned by ruff).""" + sys.stdout.write(f"{message}\n") + + +def _hooks_dir() -> Path: + """Return the hook directory under test, honouring ``AUTOPSY_HOOKS_DIR``.""" + override = os.environ.get("AUTOPSY_HOOKS_DIR") + if override: + return Path(override) + return Path.home() / ".claude" / "hooks" + + +def _gut(_text: str) -> str: + """Replace an entire hook body with a bare successful exit.""" + return _GUTTED + + +def _drop_transcript_guard(text: str) -> str: + """Remove the SessionEnd guard that requires an existing transcript.""" + return re.sub( + r'if \[\[ -z "\$transcript".*?\nfi\n', + "", + text, + flags=re.DOTALL, + ) + + +def _widen_count(text: str) -> str: + """Make a zero candidate count announce itself.""" + return text.replace("(( count > 0 ))", "(( count >= 0 ))") + + +def _drop_numeric_guard(text: str) -> str: + """Drop the count regex guard, leaving the arithmetic syntactically valid.""" + return text.replace( + '[[ "$count" =~ ^[0-9]+$ ]] && (( count > 0 ))', + "(( count > 0 ))", + ) + + +def _drop_readable_guard(text: str) -> str: + """Remove the early exit taken when the state file cannot be read.""" + return re.sub( + r'if \[\[ ! -r "\$STATE" \]\]; then\n.*?\nfi\n', + "", + text, + flags=re.DOTALL, + ) + + +def _always_announce(text: str) -> str: + """Announce unconditionally -- the defect every silence test exists to catch.""" + return re.sub(r'if \[\[ "\$count".*?; then', "if true; then", text) + + +def _unguarded_and_always_announce(text: str) -> str: + """Drop the readability guard AND announce unconditionally. + + Needed because the readability guard is redundant on its own: jq's stderr is + suppressed and an empty count already fails the arithmetic, so removing it + changes nothing observable. Only removing it together with the count + condition can reach the no-state and unreadable-state cases. + """ + return _always_announce(_drop_readable_guard(text)) + + +class Mutation(NamedTuple): + """One named defect injected into one hook.""" + + name: str + hook: str + apply: Callable[[str], str] + + +_MUTATIONS = ( + Mutation("gut-end", _END_HOOK, _gut), + Mutation("gut-start", _START_HOOK, _gut), + Mutation("end-no-transcript-guard", _END_HOOK, _drop_transcript_guard), + Mutation("start-count-ge-zero", _START_HOOK, _widen_count), + Mutation("start-no-numeric-guard", _START_HOOK, _drop_numeric_guard), + Mutation("start-no-readable-guard", _START_HOOK, _drop_readable_guard), + Mutation("start-always-announce", _START_HOOK, _always_announce), + Mutation("start-unguarded-announce", _START_HOOK, _unguarded_and_always_announce), +) + + +def _repo_root() -> Path: + """Return the repository root (this file lives in ``meta/scripts``).""" + return Path(__file__).resolve().parents[2] + + +def _run_suite(hooks_dir: Path, scratch: Path) -> set[str]: + """Run the hook test suite against ``hooks_dir``; return failing test names. + + Args: + hooks_dir: Directory holding the (possibly mutated) hooks under test. + scratch: Directory for the throwaway coverage data file. + + Returns: + The names of tests that failed or errored. + + Raises: + RuntimeError: If pytest produced no parseable outcomes at all. + """ + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + _TEST_FILE, + "-p", + "no:cacheprovider", + "-o", + "addopts=", + "-v", + "--no-header", + "--tb=no", + ], + cwd=_repo_root(), + env={ + **os.environ, + "AUTOPSY_HOOKS_DIR": str(hooks_dir), + "COVERAGE_FILE": str(scratch / ".coverage"), + }, + capture_output=True, + text=True, + check=False, + ) + outcomes = _OUTCOME.findall(result.stdout) + if not outcomes: + msg = f"pytest produced no test outcomes:\n{result.stdout[-2000:]}" + raise RuntimeError(msg) + return {name for name, outcome in outcomes if outcome in {"FAILED", "ERROR"}} + + +def _collect_test_names() -> set[str]: + """Return every ``test_*`` function name defined in the suite.""" + tree = ast.parse((_repo_root() / _TEST_FILE).read_text(encoding="utf-8")) + return { + node.name + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name.startswith("test_") + } + + +def _apply(mutation: Mutation, hooks_dir: Path, target_dir: Path) -> None: + """Copy both hooks into ``target_dir`` and inject ``mutation``. + + Raises: + RuntimeError: If the mutation left the hook text unchanged, which means + the hook was edited and the mutation no longer describes it. + """ + for hook in (_END_HOOK, _START_HOOK): + shutil.copy2(hooks_dir / hook, target_dir / hook) + target = target_dir / mutation.hook + original = target.read_text(encoding="utf-8") + mutated = mutation.apply(original) + if mutated == original: + msg = ( + f"mutation {mutation.name!r} no longer applies to {mutation.hook} -- " + "the hook text changed, so this mutation tests nothing. Update it." + ) + raise RuntimeError(msg) + target.write_text(mutated, encoding="utf-8") + target.chmod(0o755) + + +def main() -> int: + """Run every mutation and fail if any test survives them all.""" + hooks_dir = _hooks_dir() + if not all((hooks_dir / hook).is_file() for hook in (_END_HOOK, _START_HOOK)): + _emit(f"session_autopsy hooks are not deployed under {hooks_dir} -- skipping") + return 0 + + with tempfile.TemporaryDirectory() as scratch_name: + scratch = Path(scratch_name) + baseline = _run_suite(hooks_dir, scratch) + if baseline: + _emit(f"BASELINE IS RED, cannot mutate: {sorted(baseline)}") + return 1 + + killed_by: dict[str, list[str]] = {} + for mutation in _MUTATIONS: + try: + with tempfile.TemporaryDirectory() as mutant_name: + mutant_dir = Path(mutant_name) + _apply(mutation, hooks_dir, mutant_dir) + failures = _run_suite(mutant_dir, scratch) + except RuntimeError as error: + _emit(f"{mutation.name:26s} ERROR: {error}") + return 1 + _emit(f"{mutation.name:26s} killed {len(failures):2d}: {sorted(failures)}") + for test in failures: + killed_by.setdefault(test, []).append(mutation.name) + + names = _collect_test_names() + survivors = sorted(names - killed_by.keys()) + if survivors: + _emit(f"\nVACUOUS -- no mutation kills these: {survivors}") + return 1 + _emit(f"\nPASS: all {len(names)} tests killed by at least one mutation") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())