"""Tests for meta/scripts/shell_coverage_report.py. The report combines two instruments that are each wrong on their own (``docs/kcov-under-report.md``): kcov supplies the line SET but under-reports hits, and a PS4 trace supplies the executed set but cannot see a line that never ran. The arithmetic here is what makes the shell-coverage gate mean something, so the tests below pin the fail-open direction hardest: a trace must never be able to add a line that is not in the denominator. """ from __future__ import annotations import importlib.util from pathlib import Path from typing import TYPE_CHECKING import pytest if TYPE_CHECKING: from types import ModuleType _ROOT = Path(__file__).resolve().parents[2] _SCRIPT = _ROOT / "meta" / "scripts" / "shell_coverage_report.py" _XML = """ {lines} """ def _load() -> ModuleType: """Import the script by path; it lives outside any importable package.""" spec = importlib.util.spec_from_file_location("shell_coverage_report", _SCRIPT) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module @pytest.fixture(name="mod") def _mod() -> ModuleType: return _load() def _cov(tmp_path: Path, name: str, hits: dict[int, int]) -> Path: """Write a minimal kcov cov.xml tree for *name*.""" out = tmp_path / "cov" / "proc" out.mkdir(parents=True) lines = "".join(f'' for n, h in sorted(hits.items())) (out / "cov.xml").write_text(_XML.format(name=name, lines=lines), encoding="utf-8") return tmp_path / "cov" def _trace(tmp_path: Path, source: Path, numbers: list[int]) -> Path: trace = tmp_path / "trace" trace.mkdir() (trace / "case.1").write_text( "".join(f"+PS4:{source}:{n} cmd\n" for n in numbers), encoding="utf-8" ) return trace def test_kcov_hits_alone_are_used_without_a_trace( mod: ModuleType, tmp_path: Path ) -> None: """With no trace dir the result is kcov's own reading.""" cov = _cov(tmp_path, "s.sh", {1: 1, 2: 0}) covered, total, uncovered, off_set = mod._measure(cov, "s.sh", None) assert (covered, total, uncovered, off_set) == (1, 2, ["2"], []) def test_trace_rescues_a_line_kcov_missed(mod: ModuleType, tmp_path: Path) -> None: """Defect (b): a line kcov reports as 0 hits but that provably ran.""" source = tmp_path / "s.sh" source.write_text("echo one\necho two\n", encoding="utf-8") cov = _cov(tmp_path, "s.sh", {1: 1, 2: 0}) trace = _trace(tmp_path, source, [2]) covered, total, uncovered, _ = mod._measure(cov, "s.sh", trace) assert (covered, total, uncovered) == (2, 2, []) def test_continuation_lines_leave_the_denominator( mod: ModuleType, tmp_path: Path ) -> None: """Defect (a): inner lines of a multi-line quoted argument are not statements.""" source = tmp_path / "s.sh" # The real dwm_config.sh shape: the closing quote shares a line with the # rest of the command, so only the middle line is a continuation. source.write_text("perl -pe '\ns/a/b/;\n' f\necho after\n", encoding="utf-8") cov = _cov(tmp_path, "s.sh", {1: 1, 2: 0, 4: 1}) # The trace must be non-empty: it is what tells the report where the # source lives, since kcov's XML carries only a basename. trace = _trace(tmp_path, source, [1, 4]) covered, total, uncovered, _ = mod._measure(cov, "s.sh", trace) # Line 2 is inside the quote, so the denominator is 1 and 4 only. assert (covered, total, uncovered) == (2, 2, []) def test_traced_line_outside_the_line_set_is_counted_not_credited( mod: ModuleType, tmp_path: Path ) -> None: """A traced line kcov never listed cannot inflate coverage, but is reported.""" source = tmp_path / "s.sh" source.write_text("echo one\necho two\necho three\n", encoding="utf-8") cov = _cov(tmp_path, "s.sh", {1: 1}) trace = _trace(tmp_path, source, [1, 3]) covered, total, _, off_set = mod._measure(cov, "s.sh", trace) # Line 3 ran but kcov never listed it: excluded from both halves, and # named so a human can judge whether it is a statement. assert (covered, total, off_set) == (1, 1, ["3"]) def test_empty_kcov_output_reports_zero_total(mod: ModuleType, tmp_path: Path) -> None: """No instrumented lines is a distinct, reportable state.""" cov = _cov(tmp_path, "other.sh", {1: 1}) assert mod._measure(cov, "s.sh", None) == (0, 0, [], []) def test_line_without_a_number_is_skipped(mod: ModuleType, tmp_path: Path) -> None: """A malformed entry must not abort the report.""" out = tmp_path / "cov" / "proc" out.mkdir(parents=True) (out / "cov.xml").write_text( _XML.format(name="s.sh", lines=''), encoding="utf-8", ) covered, total, _, _ = mod._measure(tmp_path / "cov", "s.sh", None) assert (covered, total) == (1, 1) def test_find_source_prefers_the_trace_path(mod: ModuleType, tmp_path: Path) -> None: """kcov records only a basename, so the trace is the source of the path.""" source = tmp_path / "s.sh" source.write_text("echo one\n", encoding="utf-8") assert mod._find_source("s.sh", _trace(tmp_path, source, [1])) == source def test_find_source_ignores_a_trace_path_that_is_gone( mod: ModuleType, tmp_path: Path ) -> None: """A traced path that no longer exists falls through to the repo search.""" missing = tmp_path / "absent.sh" trace = _trace(tmp_path, missing, [1]) # Nothing in the repo is called absent.sh, so the fallback finds nothing. assert mod._find_source("absent.sh", trace) is None def test_find_source_without_a_trace_searches_the_repo(mod: ModuleType) -> None: """The fallback resolves a uniquely-named file in the repo.""" found = mod._find_source("shell_coverage_jail.sh", None) assert found is not None assert found.name == "shell_coverage_jail.sh" def test_find_source_refuses_an_ambiguous_name(mod: ModuleType) -> None: """Two files with one name yields no exclusion rather than a wrong guess.""" assert mod._find_source("run_all.sh", None) is None # -------------------------------------------------------------------------- # main(): the gate itself # -------------------------------------------------------------------------- def _run_main( mod: ModuleType, monkeypatch: pytest.MonkeyPatch, argv: list[str], ) -> None: monkeypatch.setattr("sys.argv", ["shell_coverage_report.py", *argv]) mod.main() def test_main_prints_the_summary( mod: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """A fully covered subject prints its percentage and passes the gate.""" cov = _cov(tmp_path, "s.sh", {1: 1, 2: 1}) _run_main(mod, monkeypatch, [str(cov), "s.sh", "100"]) assert "s.sh: 2/2 lines = 100.00%" in capsys.readouterr().out def test_main_lists_uncovered_lines( mod: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """A gap is named, because the numbers alone do not help close it.""" cov = _cov(tmp_path, "s.sh", {1: 1, 2: 0}) with pytest.raises(SystemExit): _run_main(mod, monkeypatch, [str(cov), "s.sh", "100"]) assert "uncovered: 2" in capsys.readouterr().out def test_main_notes_lines_outside_the_line_set( mod: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """A traced line kcov never listed is surfaced as a third-defect signal.""" source = tmp_path / "s.sh" source.write_text("echo one\necho two\n", encoding="utf-8") cov = _cov(tmp_path, "s.sh", {1: 1}) trace = _trace(tmp_path, source, [1, 2]) _run_main(mod, monkeypatch, [str(cov), "s.sh", "100", str(trace)]) assert "outside kcov's line set" in capsys.readouterr().out def test_main_exits_when_below_the_minimum( mod: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Below the bar is a non-zero exit, which is what makes this a gate.""" cov = _cov(tmp_path, "s.sh", {1: 1, 2: 0}) with pytest.raises(SystemExit) as excinfo: _run_main(mod, monkeypatch, [str(cov), "s.sh", "100"]) assert "below the required" in str(excinfo.value) def test_main_exits_when_nothing_was_instrumented( mod: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Zero instrumented lines is an error, never a vacuous 100%.""" cov = _cov(tmp_path, "other.sh", {1: 1}) with pytest.raises(SystemExit) as excinfo: _run_main(mod, monkeypatch, [str(cov), "s.sh", "100"]) assert "instrumented no lines" in str(excinfo.value)