testsAndMisc/python_pkg/artgate/tests/test_cli.py
Krzysztof kuhy Rudnicki da8ca6d827
Some checks are pending
Pre-commit checks / pre-commit (push) Waiting to run
Python tests / test (push) Waiting to run
Shell tests / shell-tests (push) Waiting to run
Shell tests / arch-tests (push) Waiting to run
artgate: deterministic exit-code gates for generated game art
Adds the route-independent spine of the planned art-generation bake-off: a
harness that decides whether generated art is well-formed, so "non-sloppy"
becomes machine-checkable instead of a judgement call.

Findings that shaped the design, all measured rather than assumed:

- Integer-scale invariance separates true pixel art (impurity 0.0000) from
  interpolated fakes (1.0000) exactly -- but passes trivially at scale 1, so
  it must never be trusted alone on native-resolution output.
- Gate ORDER is load-bearing. A soft-alpha image has no fully-opaque pixels,
  so an opaque-only colour count returns 0 and silently passes. ALPHA_NOT_BINARY
  now runs first and COLOR_COUNT fails closed on an empty opaque set.
- Counting must be over opaque RGB only: transparent pixels retain arbitrary
  RGB and inflated a naive RGBA count, rejecting correct art at 18/16.
- Declaring a gate was not enough to run it. hand_painted.toml shipped
  declaring SILHOUETTE_LOW_CONTRAST with no implementation, so it parsed, ran
  nothing, and reported a pass. load_spec now fails closed on unimplemented
  gates, result-only codes, and gates missing their config.

Thresholds are calibrated against 250 genuine 32x32 CC0 icons, not guessed:
p50=7 p90=13 p99=26 opaque colours; cap 32 rejects 0.0% of real art while raw
generator output measured 34403 colours. An initial cap of 16 wrongly rejected
a real DCSS icon -- re-derived from the corpus rather than nudged.

Verified: 57 tests, 100% branch coverage, zero suppressions, all hooks pass.
58/60 (96.7%) of real CC0 icons pass the full stack; both failures confirmed
true positives.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T9rQ12yZ7sJxiPQTCS5dUF
2026-08-15 22:20:36 +02:00

168 lines
5.4 KiB
Python

"""CLI tests, covering all three exit codes.
The three-valued exit code is the contract the whole bake-off rests on, so
each value is asserted explicitly: a harness fault must never be recorded as
an art rejection.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import numpy as np
from PIL import Image
import pytest
from python_pkg.artgate.cli import evaluate, main
from python_pkg.artgate.codes import (
EXIT_GATE_FAILURE,
EXIT_HARNESS_ERROR,
EXIT_PASS,
Code,
)
from python_pkg.artgate.spec import TargetSpec
if TYPE_CHECKING:
from pathlib import Path
SPEC = """
name = "t"
canvas = [4, 4]
max_colors = 4
alpha_binary = true
palette = ["#0a141e", "#28323c"]
gates = [
"ALPHA_NOT_BINARY", "TOO_MANY_COLORS", "OFF_PALETTE",
"CANVAS_SIZE", "SCALE_NOT_INVARIANT",
]
"""
def _write_png(path: Path, rgb: tuple[int, int, int], size: int = 4) -> Path:
"""Write a uniform opaque PNG.
Args:
path: Destination file.
rgb: The fill colour.
size: Square edge length.
Returns:
The written path.
"""
arr = np.zeros((size, size, 4), dtype=np.uint8)
arr[:, :, :3] = rgb
arr[:, :, 3] = 255
Image.fromarray(arr, "RGBA").save(path)
return path
@pytest.fixture(name="spec_file")
def fixture_spec_file(tmp_path: Path) -> Path:
"""Write the shared test spec.
Args:
tmp_path: Pytest temporary directory.
Returns:
Path to the spec.
"""
path = tmp_path / "t.toml"
path.write_text(SPEC, encoding="utf-8")
return path
class TestExitCodes:
"""Each exit code is reachable and means exactly one thing."""
def test_pass(
self, tmp_path: Path, spec_file: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""Conforming art exits 0."""
img = _write_png(tmp_path / "ok.png", (10, 20, 30))
assert main([str(img), "--spec", str(spec_file)]) == EXIT_PASS
assert "PASS" in capsys.readouterr().out
def test_gate_failure(
self, tmp_path: Path, spec_file: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""Off-palette art exits 1 and names the code."""
img = _write_png(tmp_path / "bad.png", (200, 10, 10))
assert main([str(img), "--spec", str(spec_file)]) == EXIT_GATE_FAILURE
assert "OFF_PALETTE" in capsys.readouterr().out
def test_missing_spec_is_harness_error(
self, tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""A missing spec exits 2, not 1."""
img = _write_png(tmp_path / "x.png", (10, 20, 30))
code = main([str(img), "--spec", str(tmp_path / "absent.toml")])
assert code == EXIT_HARNESS_ERROR
assert "artgate:" in capsys.readouterr().err
def test_unreadable_image_is_harness_error(
self, tmp_path: Path, spec_file: Path
) -> None:
"""A non-image file exits 2, not 1."""
junk = tmp_path / "junk.png"
junk.write_text("not an image", encoding="utf-8")
code = main([str(junk), "--spec", str(spec_file)])
assert code == EXIT_HARNESS_ERROR
def test_json_output(
self, tmp_path: Path, spec_file: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""JSON mode emits the machine-readable verdict."""
img = _write_png(tmp_path / "ok.png", (10, 20, 30))
main([str(img), "--spec", str(spec_file), "--json"])
out = capsys.readouterr().out
assert '"exit": 0' in out
assert '"style": "t"' in out
class TestEvaluate:
"""Gate selection is driven by the spec, not by hardcoded branching."""
def test_canvas_mismatch_reported(self, tmp_path: Path) -> None:
"""A wrong-sized canvas is caught."""
arr = np.zeros((8, 8, 4), dtype=np.uint8)
arr[:, :, 3] = 255
spec = TargetSpec(name="s", canvas=(4, 4), gates=frozenset({Code.CANVAS_SIZE}))
assert evaluate(arr, spec) == [Code.CANVAS_SIZE]
def test_gates_not_declared_are_not_run(self) -> None:
"""An off-palette image passes when the palette gate is not declared."""
arr = np.zeros((2, 2, 4), dtype=np.uint8)
arr[:, :, :3] = (99, 99, 99)
arr[:, :, 3] = 255
spec = TargetSpec(
name="s",
palette=frozenset({(1, 2, 3)}),
gates=frozenset({Code.CANVAS_SIZE}),
)
assert evaluate(arr, spec) == []
def test_codes_are_deduplicated(self) -> None:
"""A code raised by two gates is reported once."""
arr = np.zeros((2, 2, 4), dtype=np.uint8)
spec = TargetSpec(
name="s",
max_colors=4,
palette=frozenset({(1, 2, 3)}),
gates=frozenset({Code.TOO_MANY_COLORS, Code.OFF_PALETTE}),
)
assert evaluate(arr, spec) == [Code.NO_OPAQUE_PIXELS]
def test_silhouette_gate_runs_when_declared(self) -> None:
"""A near-white asset fails the silhouette gate.
Regression: ``SILHOUETTE_LOW_CONTRAST`` was declarable in a spec but
had no implementation, so it parsed cleanly, ran nothing and reported
a pass. ``load_spec`` now rejects undeclarable gates, and this asserts
the gate genuinely executes once declared.
"""
arr = np.zeros((8, 8, 4), dtype=np.uint8)
arr[:, :, :3] = 250
arr[:, :, 3] = 255
spec = TargetSpec(name="s", gates=frozenset({Code.SILHOUETTE_LOW_CONTRAST}))
assert evaluate(arr, spec) == [Code.SILHOUETTE_LOW_CONTRAST]