mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 20:27:02 +01:00
meta: make this session's 65 mutations reproducible from the repo
Four evidence files committed today cite mutation runs -- 18, 13, 17 and
17 killed -- that lived in a scratch directory. Evidence pointing at a run
nobody can repeat is not evidence, so the lists move into the repo behind a
spec-driven runner.
Mutations are literal find/replace strings in JSON, not sed expressions.
The shell version this replaces escaped each pattern twice and used | as
its field separator, which collided with || and reported six mutations as
SURVIVED when they had never been applied at all -- a false negative in the
one tool whose entire job is catching false negatives.
Two deliberate choices follow from that:
- a no-op (find matched nothing) counts as a FAILURE, not a skip. A stale
spec entry otherwise stops testing its behaviour without saying so.
- a timeout counts as killed: a mutation that hangs the suite has still
been detected by it.
The conversion itself exercised the first rule -- it reported 8 no-ops from
mangled escapes rather than quietly scoring them as killed. Each was fixed
against the real source line.
All four specs now reproduce their committed figures exactly, and the
working tree is byte-clean after every run.
Not wired into pre-commit: it runs the whole suite once per mutation, which
is minutes, not seconds.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
This commit is contained in:
parent
0a07684cca
commit
35ddf4114f
61
docs/superpowers/evidence/mutate-shell-tool-2026-08-17.json
Normal file
61
docs/superpowers/evidence/mutate-shell-tool-2026-08-17.json
Normal file
@ -0,0 +1,61 @@
|
||||
{
|
||||
"intent": "Make the mutation-testing claims in this session's four evidence files reproducible from the repo. They cited runs from a scratch directory that will not survive the session, leaving committed evidence pointing at something nobody could repeat.",
|
||||
"scope": [
|
||||
"meta/scripts/mutate_shell.py - new: a spec-driven mutation runner",
|
||||
"meta/scripts/fixtures/mutations/{monitor,backup_capture,curfew_net,hosts_libs}.json - new: the 65 mutations already used",
|
||||
"Non-goal: wiring this into pre-commit. It runs the full shell suite once per mutation, which is far too slow for a commit hook.",
|
||||
"Non-goal: changing any subject or test; this only relocates and formalises checks already performed."
|
||||
],
|
||||
"changes": [
|
||||
"Declared mutations in JSON with literal find/replace strings rather than sed expressions. The shell version escaped each pattern twice and used | as a field separator, which collided with || and silently reported six mutations as survivors when they had never been applied — a false negative in the exact tool meant to catch false negatives.",
|
||||
"Counted a no-op (find matched nothing) as a failure rather than ignoring it. A stale spec entry otherwise stops testing its behaviour without ever saying so, which is the same silent-pass failure mode the tool exists to detect.",
|
||||
"Restored the subject in a finally block, so an interrupted run cannot leave a deliberately broken file in the working tree.",
|
||||
"Treated a timeout as killed: a mutation that makes the suite hang has still been detected by it."
|
||||
],
|
||||
"verification": [
|
||||
{
|
||||
"command": "python3 meta/scripts/mutate_shell.py meta/scripts/fixtures/mutations/monitor.json",
|
||||
"result": "pass",
|
||||
"evidence": "18 killed, 0 not killed — matching the figure in monitor-coverage-backfill-2026-08-17.json."
|
||||
},
|
||||
{
|
||||
"command": "python3 meta/scripts/mutate_shell.py meta/scripts/fixtures/mutations/backup_capture.json",
|
||||
"result": "pass",
|
||||
"evidence": "13 killed, 0 not killed — matching phone-backup-capture-split-2026-08-17.json."
|
||||
},
|
||||
{
|
||||
"command": "python3 meta/scripts/mutate_shell.py meta/scripts/fixtures/mutations/curfew_net.json",
|
||||
"result": "pass",
|
||||
"evidence": "17 killed, 0 not killed — matching curfew-net-split-2026-08-17.json."
|
||||
},
|
||||
{
|
||||
"command": "python3 meta/scripts/mutate_shell.py meta/scripts/fixtures/mutations/hosts_libs.json",
|
||||
"result": "pass",
|
||||
"evidence": "17 killed, 0 not killed — matching hosts-enforcer-split-2026-08-17.json."
|
||||
},
|
||||
{
|
||||
"command": "git status --short after every run",
|
||||
"result": "pass",
|
||||
"evidence": "Clean. The restore path returns each subject byte-for-byte; an earlier shell version left a mode-only diff behind, which is why this one uses copy2 and an unconditional finally."
|
||||
},
|
||||
{
|
||||
"command": "pre-commit run --files meta/scripts/mutate_shell.py meta/scripts/fixtures/mutations/*.json",
|
||||
"result": "pass",
|
||||
"evidence": "Clean, including ruff, mypy and pylint. Ruff's autofixer removed a suppression during the first run; the tool was re-run afterwards and still reports 13 killed on backup_capture, so the autofix changed nothing behavioural."
|
||||
},
|
||||
{
|
||||
"command": "first conversion run of all four specs",
|
||||
"result": "pass",
|
||||
"evidence": "Reported 8 no-ops rather than silently passing, caused by the sed-to-literal conversion mangling escapes. Each was corrected against the actual source line and re-run. That the tool surfaced them rather than counting them as killed is the behaviour being relied on."
|
||||
}
|
||||
],
|
||||
"risks": [
|
||||
"The specs pin literal source text, so any edit to a mutated line makes its entry a no-op. That is reported loudly rather than passing quietly, but it does mean the specs need updating alongside the code they target.",
|
||||
"This is not enforced by any hook. Nothing prevents a future shell library from being added with a coverage number and no mutation spec; the discipline remains a convention.",
|
||||
"A full spec run takes minutes, since it executes the whole suite once per mutation. It is a deliberate, occasional check, not something to put in a tight loop."
|
||||
],
|
||||
"rollback": [
|
||||
"git revert the commit. The four evidence files then cite mutation runs with no reproducible path, which is the state this commit exists to fix — prefer fixing forward.",
|
||||
"After rollback, nothing else breaks: no hook, test or script references the tool."
|
||||
]
|
||||
}
|
||||
84
meta/scripts/fixtures/mutations/backup_capture.json
Normal file
84
meta/scripts/fixtures/mutations/backup_capture.json
Normal file
@ -0,0 +1,84 @@
|
||||
{
|
||||
"test": "phone_focus_mode/lib/tests/test_backup_capture.sh",
|
||||
"timeout": 180,
|
||||
"mutations": [
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "sed 's/^package://'",
|
||||
"replace": "cat",
|
||||
"describe": "stop stripping the package: prefix"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "tr -d '\\r'",
|
||||
"replace": "cat",
|
||||
"describe": "stop stripping carriage returns"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "comm -13",
|
||||
"replace": "comm -12",
|
||||
"describe": "invert the purged-package derivation"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "pm list packages -3",
|
||||
"replace": "pm list packages",
|
||||
"describe": "dump all packages as third-party"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "mapfile -t pkgs <\"${dir}/packages-third-party.txt\"",
|
||||
"replace": "read -r pkgs <\"${dir}/packages-third-party.txt\"",
|
||||
"describe": "reintroduce the stdin-draining apk bug"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "for path in \"${paths[@]}\"; do",
|
||||
"replace": "for path in \"${paths[0]}\"; do",
|
||||
"describe": "pull only the first split of a split app"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "_warn \"no APK path for ${pkg}\"",
|
||||
"replace": ":",
|
||||
"describe": "silence the missing-APK-path warning"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "_warn \"could not pull ${base} for ${pkg}\"",
|
||||
"replace": ":",
|
||||
"describe": "silence the failed-pull warning"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "_warn \"partial or failed pull: ${d}\"",
|
||||
"replace": ":",
|
||||
"describe": "silence the sdcard partial-pull warning"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "if adb_cmd shell \"test -d /sdcard/${d}\"",
|
||||
"replace": "if ! adb_cmd shell \"test -d /sdcard/${d}\"",
|
||||
"describe": "invert the sdcard directory probe"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "for ns in system secure global; do",
|
||||
"replace": "for ns in system; do",
|
||||
"describe": "drop two settings namespaces"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "grep -qx 'com.celzero.bravedns'",
|
||||
"replace": "grep -qvx 'com.celzero.bravedns'",
|
||||
"describe": "invert the RethinkDNS detection"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/backup_capture.sh",
|
||||
"find": "mkdir -p \"${dir}/sdcard/keepass\"",
|
||||
"replace": ":",
|
||||
"describe": "stop creating the keepass directory"
|
||||
}
|
||||
]
|
||||
}
|
||||
108
meta/scripts/fixtures/mutations/curfew_net.json
Normal file
108
meta/scripts/fixtures/mutations/curfew_net.json
Normal file
@ -0,0 +1,108 @@
|
||||
{
|
||||
"test": "phone_focus_mode/lib/tests/test_curfew_net.sh",
|
||||
"timeout": 180,
|
||||
"mutations": [
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "-I OUTPUT 1 -j",
|
||||
"replace": "-I OUTPUT 2 -j",
|
||||
"describe": "stop pinning the OUTPUT jump at position 1"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "iptw \"$ipt\" -N \"$CURFEW_NET_IPT_CHAIN\" 2>/dev/null || return 1",
|
||||
"replace": "iptw \"$ipt\" -N \"$CURFEW_NET_IPT_CHAIN\" 2>/dev/null || true",
|
||||
"describe": "swallow a failed chain creation"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "iptw \"$ipt\" -I OUTPUT 1 -j \"$CURFEW_NET_IPT_CHAIN\" 2>/dev/null || return 1",
|
||||
"replace": "iptw \"$ipt\" -I OUTPUT 1 -j \"$CURFEW_NET_IPT_CHAIN\" 2>/dev/null || true",
|
||||
"describe": "swallow a failed jump insert"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "iptw \"$ipt\" -F \"$CURFEW_NET_IPT_CHAIN\" 2>/dev/null || return 1",
|
||||
"replace": "iptw \"$ipt\" -F \"$CURFEW_NET_IPT_CHAIN\" 2>/dev/null || true",
|
||||
"describe": "swallow a failed flush and layer onto stale rules"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "-o lo -j ACCEPT",
|
||||
"replace": "-o lo -j REJECT",
|
||||
"describe": "reject loopback"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "--state ESTABLISHED,RELATED -j ACCEPT",
|
||||
"replace": "--state ESTABLISHED,RELATED -j REJECT",
|
||||
"describe": "reject established flows"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "--uid-owner 0 -j ACCEPT",
|
||||
"replace": "--uid-owner 0 -j REJECT",
|
||||
"describe": "cut off root"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "-p udp --dport 53 -j ACCEPT",
|
||||
"replace": "-p udp --dport 5353 -j ACCEPT",
|
||||
"describe": "allow the wrong DNS port"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "--uid-owner 10000-19999 -j REJECT",
|
||||
"replace": "--uid-owner 10000-19999 -j ACCEPT",
|
||||
"describe": "let every app through instead of cutting it off"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "case \"$uid\" in ''|*[!0-9]*) continue ;; esac",
|
||||
"replace": ":",
|
||||
"describe": "pass non-numeric uids to iptables"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "$1 == p {",
|
||||
"replace": "$1 != p {",
|
||||
"describe": "invert the package-to-uid match"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "mv \"$CURFEW_NET_UID_CACHE.tmp\" \"$CURFEW_NET_UID_CACHE\"",
|
||||
"replace": "cp \"$CURFEW_NET_UID_CACHE.tmp\" \"$CURFEW_NET_UID_CACHE\"",
|
||||
"describe": "leave the uid cache temp file behind"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "rebuilds=$((rebuilds + 1))",
|
||||
"replace": ":",
|
||||
"describe": "stop counting net_hold rebuilds"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "[ \"${CURFEW_NET_ENABLED:-0}\" = \"1\" ] || return 0",
|
||||
"replace": ":",
|
||||
"describe": "ignore the net-layer disable switch"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "NET_BUILT=1",
|
||||
"replace": ":",
|
||||
"describe": "never mark the chain as built"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "iptw \"$ipt\" -X \"$CURFEW_NET_IPT_CHAIN\" 2>/dev/null || true",
|
||||
"replace": ":",
|
||||
"describe": "leave the chain behind on teardown"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/curfew_net.sh",
|
||||
"find": "rm -f \"$map\"",
|
||||
"replace": ":",
|
||||
"describe": "leak the uid map temp file"
|
||||
}
|
||||
]
|
||||
}
|
||||
108
meta/scripts/fixtures/mutations/hosts_libs.json
Normal file
108
meta/scripts/fixtures/mutations/hosts_libs.json
Normal file
@ -0,0 +1,108 @@
|
||||
{
|
||||
"test": "phone_focus_mode/lib/tests/test_hosts_libs.sh",
|
||||
"timeout": 180,
|
||||
"mutations": [
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_mount.sh",
|
||||
"find": "[ -n \"$target_hash\" ] && [ \"$target_hash\" = \"$canonical_hash\" ]",
|
||||
"replace": "true",
|
||||
"describe": "call any target correctly mounted"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_mount.sh",
|
||||
"find": "[ -n \"$target_hash\" ] && ",
|
||||
"replace": "",
|
||||
"describe": "treat a missing target as mounted"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_mount.sh",
|
||||
"find": "[ \"$attempts\" -ge 5 ]",
|
||||
"replace": "[ \"$attempts\" -ge 500 ]",
|
||||
"describe": "raise the unmount give-up threshold"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_mount.sh",
|
||||
"find": "mount -o remount,ro \"$system_mount\"",
|
||||
"replace": ":",
|
||||
"describe": "leave /system mounted read-write"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_mount.sh",
|
||||
"find": "chattr +i \"$HOSTS_TARGET\"",
|
||||
"replace": ":",
|
||||
"describe": "never re-lock the hosts target"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_mount.sh",
|
||||
"find": "log \"Bind mount reported success but target still mismatches - unmounting\"",
|
||||
"replace": ":",
|
||||
"describe": "silence the no-op bind detection"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_mount.sh",
|
||||
"find": "\n return 1\n}",
|
||||
"replace": "\n return 0\n}",
|
||||
"describe": "claim success when nothing fixed the target"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "if [ \"$module_hash\" != \"$canonical_hash\" ]; then",
|
||||
"replace": "if true; then",
|
||||
"describe": "rewrite the module on every poll"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "\n protect_magisk_module\n}",
|
||||
"replace": "\n :\n}",
|
||||
"describe": "stop re-locking the module directory"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "for marker in disable remove update; do",
|
||||
"replace": "for marker in disable; do",
|
||||
"describe": "ignore the remove and update markers"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "removed=$((removed + 1))",
|
||||
"replace": ":",
|
||||
"describe": "stop counting removed markers"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "chattr +i \"$module_dir\"",
|
||||
"replace": ":",
|
||||
"describe": "never lock the module directory"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "[ -f \"${profile_dir}prefs.js\" ] || continue",
|
||||
"replace": ":",
|
||||
"describe": "write to non-profile directories"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "network.trr.mode\", 5",
|
||||
"replace": "network.trr.mode\", 0",
|
||||
"describe": "set the DoH pref to the wrong mode"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "chattr +i \"$HOSTS_CANONICAL\"",
|
||||
"replace": ":",
|
||||
"describe": "leave the canonical mutable"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "if [ \"$current_pid\" = \"$last_pid\" ]; then",
|
||||
"replace": "if false; then",
|
||||
"describe": "restart netd on every tick"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/hosts_magisk.sh",
|
||||
"find": "echo \"${new_pid:-$current_pid}\" > \"$stamp_file\"",
|
||||
"replace": ":",
|
||||
"describe": "never stamp the netd pid"
|
||||
}
|
||||
]
|
||||
}
|
||||
114
meta/scripts/fixtures/mutations/monitor.json
Normal file
114
meta/scripts/fixtures/mutations/monitor.json
Normal file
@ -0,0 +1,114 @@
|
||||
{
|
||||
"test": "phone_focus_mode/lib/tests/test_monitor.sh",
|
||||
"timeout": 180,
|
||||
"mutations": [
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor.sh",
|
||||
"find": "escaped=${escaped//\\\"/\\\\\\\"}",
|
||||
"replace": ":",
|
||||
"describe": "drop the JSON quote escaping"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor.sh",
|
||||
"find": "escaped=${escaped//$'\\n'/\\\\n}",
|
||||
"replace": ":",
|
||||
"describe": "drop the newline escaping"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor.sh",
|
||||
"find": "(( missing_count >= FORMAT_DETECTION_MIN_MISSING ))",
|
||||
"replace": "(( missing_count > FORMAT_DETECTION_MIN_MISSING ))",
|
||||
"describe": "off-by-one the format threshold"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor.sh",
|
||||
"find": "adb_root_shell \"kill -0 ${pid} >/dev/null 2>&1\" >/dev/null 2>&1 || return 1",
|
||||
"replace": "adb_root_shell \"kill -0 ${pid} >/dev/null 2>&1\" >/dev/null 2>&1 || return 0",
|
||||
"describe": "make pid_matches_script always succeed"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor.sh",
|
||||
"find": "printf '%s' \"${candidate}\"",
|
||||
"replace": "printf ''",
|
||||
"describe": "make hosts candidate resolution return empty"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_health.sh",
|
||||
"find": "status=\"warn\"",
|
||||
"replace": "status=\"ok\"",
|
||||
"describe": "silence every health warning"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_health.sh",
|
||||
"find": "status=\"fatal\"",
|
||||
"replace": "status=\"ok\"",
|
||||
"describe": "silence the format-indicator fatal"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_health.sh",
|
||||
"find": "(( level < BATTERY_WARN_BELOW ))",
|
||||
"replace": "(( level > BATTERY_WARN_BELOW ))",
|
||||
"describe": "invert the battery threshold"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_health.sh",
|
||||
"find": "(( free_mb < STORAGE_WARN_BELOW_MB ))",
|
||||
"replace": "(( free_mb > STORAGE_WARN_BELOW_MB ))",
|
||||
"describe": "invert the storage threshold"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_health.sh",
|
||||
"find": "free_mb=$((free_kb / 1024))",
|
||||
"replace": "free_mb=$((free_kb / 512))",
|
||||
"describe": "wrong KB-to-MB divisor"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_health.sh",
|
||||
"find": "\"${daemon_name} is NOT running\" \"true\"",
|
||||
"replace": "\"${daemon_name} is NOT running\" \"false\"",
|
||||
"describe": "mark a dead daemon unrepairable"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_policy.sh",
|
||||
"find": "\"hosts_integrity\" \"fatal\"",
|
||||
"replace": "\"hosts_integrity\" \"ok\"",
|
||||
"describe": "call a missing canonical hosts file ok"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_policy.sh",
|
||||
"find": "\"${expected_hash}\" == \"${actual_hash}\"",
|
||||
"replace": "\"${expected_hash}\" != \"${actual_hash}\"",
|
||||
"describe": "invert the hosts hash comparison"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_policy.sh",
|
||||
"find": "chain_present=\"yes\"",
|
||||
"replace": "chain_present=\"no\"",
|
||||
"describe": "never observe the DNS chain"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_policy.sh",
|
||||
"find": "\"launcher_state\" \"fatal\"",
|
||||
"replace": "\"launcher_state\" \"warn\"",
|
||||
"describe": "downgrade the missing-launcher fatal"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_policy.sh",
|
||||
"find": "\"launcher_state\" \"error\"",
|
||||
"replace": "\"launcher_state\" \"ok\"",
|
||||
"describe": "call a launcher mismatch ok"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_policy.sh",
|
||||
"find": "\"boot_persistence\" \"fatal\"",
|
||||
"replace": "\"boot_persistence\" \"ok\"",
|
||||
"describe": "call a missing boot script ok"
|
||||
},
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor_checks_policy.sh",
|
||||
"find": "\"companion_app\" \"warn\"",
|
||||
"replace": "\"companion_app\" \"ok\"",
|
||||
"describe": "call a missing companion app ok"
|
||||
}
|
||||
]
|
||||
}
|
||||
173
meta/scripts/mutate_shell.py
Executable file
173
meta/scripts/mutate_shell.py
Executable file
@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""Mutation-test a shell library against its test script.
|
||||
|
||||
A test that passes proves nothing on its own. kcov measures *lines*, not
|
||||
branches, so ``[ -n "$x" ] && thing`` counts as covered when only one side
|
||||
ever runs -- and a suite can reach 100% while asserting nothing that would
|
||||
break if the code were wrong. This applies the check that does discriminate:
|
||||
break the subject, require the suite to fail, restore.
|
||||
|
||||
Mutations are declared in a JSON spec so a run is reproducible from the repo
|
||||
rather than from a scratch directory::
|
||||
|
||||
{
|
||||
"test": "phone_focus_mode/lib/tests/test_monitor.sh",
|
||||
"timeout": 180,
|
||||
"mutations": [
|
||||
{
|
||||
"file": "phone_focus_mode/lib/monitor.sh",
|
||||
"find": "status=\\"warn\\"",
|
||||
"replace": "status=\\"ok\\"",
|
||||
"describe": "silence every health warning"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
``find`` is a literal string, not a regex: a sed-style pattern has to be
|
||||
escaped twice (once for JSON, once for the shell) and the delimiter collides
|
||||
with ``||``, which silently produced six false survivors when this started
|
||||
life as a shell script.
|
||||
|
||||
Three outcomes, and only one of them is success:
|
||||
|
||||
* **killed** -- the suite failed, so the assertions cover this behaviour.
|
||||
* **survived** -- the suite passed with the code broken. Either the assertion
|
||||
is missing, or the mutation is *equivalent* and the code it changes is dead.
|
||||
Both are findings; neither is resolved by deleting the mutation.
|
||||
* **no-op** -- ``find`` matched nothing, so the spec is stale. Counted as a
|
||||
failure, because a mutation that cannot apply silently stops testing.
|
||||
|
||||
Exit status is 0 only when every mutation was killed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mutation:
|
||||
"""One edit to apply to one file, with a human-readable description."""
|
||||
|
||||
file: Path
|
||||
find: str
|
||||
replace: str
|
||||
describe: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Spec:
|
||||
"""A test script plus the mutations it is expected to catch."""
|
||||
|
||||
test: Path
|
||||
mutations: tuple[Mutation, ...]
|
||||
timeout: int
|
||||
|
||||
|
||||
def _load_spec(path: Path, root: Path) -> Spec:
|
||||
"""Parse a mutation spec, resolving every path relative to ``root``."""
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
mutations = tuple(
|
||||
Mutation(
|
||||
file=root / entry["file"],
|
||||
find=entry["find"],
|
||||
replace=entry["replace"],
|
||||
describe=entry["describe"],
|
||||
)
|
||||
for entry in data["mutations"]
|
||||
)
|
||||
if not mutations:
|
||||
message = f"{path}: spec declares no mutations"
|
||||
raise ValueError(message)
|
||||
return Spec(
|
||||
test=root / data["test"],
|
||||
mutations=mutations,
|
||||
timeout=int(data.get("timeout", 180)),
|
||||
)
|
||||
|
||||
|
||||
def _suite_passes(test: Path, root: Path, timeout: int) -> bool:
|
||||
"""Run the test script, returning whether it exited 0."""
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
["/usr/bin/env", "bash", str(test)],
|
||||
cwd=root,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
# A hang is not a pass: the mutation broke the suite badly enough that
|
||||
# it never finished, which still means the behaviour is covered.
|
||||
return False
|
||||
return completed.returncode == 0
|
||||
|
||||
|
||||
def _apply(mutation: Mutation) -> bool:
|
||||
"""Apply one mutation in place. Returns False when it matched nothing."""
|
||||
original = mutation.file.read_text(encoding="utf-8")
|
||||
mutated = original.replace(mutation.find, mutation.replace)
|
||||
if mutated == original:
|
||||
return False
|
||||
mutation.file.write_text(mutated, encoding="utf-8")
|
||||
return True
|
||||
|
||||
|
||||
def _run_one(mutation: Mutation, spec: Spec, root: Path) -> str:
|
||||
"""Apply, run, restore. Returns 'killed', 'survived' or 'no-op'."""
|
||||
backup = mutation.file.with_suffix(mutation.file.suffix + ".mutbak")
|
||||
shutil.copy2(mutation.file, backup)
|
||||
try:
|
||||
if not _apply(mutation):
|
||||
return "no-op"
|
||||
return "survived" if _suite_passes(spec.test, root, spec.timeout) else "killed"
|
||||
finally:
|
||||
# Restore unconditionally: a crash mid-run must not leave the working
|
||||
# tree holding a deliberately broken subject.
|
||||
shutil.move(str(backup), str(mutation.file))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run every mutation in the spec and report the tally."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("spec", type=Path, help="path to the mutation spec JSON")
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
type=Path,
|
||||
default=Path.cwd(),
|
||||
help="repo root that spec paths are relative to (default: cwd)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
root = args.root.resolve()
|
||||
spec = _load_spec(args.spec, root)
|
||||
|
||||
killed = 0
|
||||
failures: list[str] = []
|
||||
|
||||
for mutation in spec.mutations:
|
||||
outcome = _run_one(mutation, spec, root)
|
||||
if outcome == "killed":
|
||||
killed += 1
|
||||
else:
|
||||
failures.append(f"{outcome.upper()}: {mutation.describe}")
|
||||
sys.stdout.write(f"{outcome:>8}: {mutation.describe}\n")
|
||||
|
||||
sys.stdout.write(
|
||||
f"\nMutations: {killed} killed, {len(failures)} not killed "
|
||||
f"(of {len(spec.mutations)})\n",
|
||||
)
|
||||
for failure in failures:
|
||||
sys.stdout.write(f" {failure}\n")
|
||||
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue
Block a user