Narrow the live enforcer's catch-all handlers

Every except Exception in the daemon, ctl socket, web server and job
runner now catches FAIL_SAFE_ERRORS (OSError, ValueError, TypeError,
LookupError, RuntimeError, SubprocessError). A bug outside it is loud:
the ctl thread dies with its slot freed, http.server reports it, a job
writes its failed result and re-raises.

Todo: 9b9342ba

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Krzysztof Rudnicki 2026-11-12 08:29:03 +01:00
parent db44ca5b0b
commit 444f619e6d
15 changed files with 210 additions and 37 deletions

View File

@ -41,6 +41,7 @@ from steam_backlog_enforcer._ctl_wire import (
read_line,
socket_is_live,
)
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
if TYPE_CHECKING:
from pathlib import Path
@ -172,8 +173,8 @@ class CtlServer:
line = read_line(conn)
if line is not None:
self._write(conn, self.dispatch(line, uid))
# Catch-all on purpose: a dying connection must not kill its thread.
except Exception: # pylint: disable=broad-exception-caught
# A dying connection (timeout, hang-up) must not kill its thread.
except OSError:
logger.debug("Control connection dropped", exc_info=True)
finally:
conn.close()
@ -186,14 +187,15 @@ class CtlServer:
conn.sendall(payload)
def dispatch(self, line: bytes, uid: int) -> bytes:
"""Turn a request line into a response line. Never raises."""
"""Turn a request line into a response line; op failures become errors."""
try:
return ok_response(self.run(line, uid))
except CtlError as exc:
logger.info("Control socket: refused (%s)", exc.code)
return error_response(exc)
# Catch-all on purpose: any op bug answers "internal error", never raises.
except Exception: # pylint: disable=broad-exception-caught
# A failing op answers "internal error"; a bug outside the tuple ends
# only this connection's thread (``handle`` still frees its slot).
except FAIL_SAFE_ERRORS:
logger.exception("Control socket op crashed")
return error_response(CtlError(OP_FAILED, "internal error"))

View File

@ -0,0 +1,30 @@
"""The exceptions a fail-safe handler absorbs; anything else propagates.
No handler in the live enforcer catches ``Exception`` (kuhy's call, todo
9b9342ba). A handler that must outlive a failing input or op catches this
tuple instead:
* ``OSError``: files, sockets, ``/proc``, signals (incl. timeouts, a peer
that hung up, ``ProcessLookupError``);
* ``ValueError``: malformed data, incl. ``json`` and Unicode decode errors;
* ``TypeError`` and ``LookupError``: a payload of the wrong shape;
* ``RuntimeError``: incl. ``RecursionError``, which leisure_watch has raised;
* ``subprocess.SubprocessError``: a helper that failed or timed out.
A programming bug outside it (``AttributeError``, ``NameError``, ...) is loud:
it reaches the thread's or the process's own error reporting.
"""
from __future__ import annotations
import subprocess
from typing import Final
FAIL_SAFE_ERRORS: Final[tuple[type[Exception], ...]] = (
OSError,
ValueError,
TypeError,
LookupError,
RuntimeError,
subprocess.SubprocessError,
)

View File

@ -11,9 +11,11 @@ import difflib
import logging
import re
from steam_backlog_enforcer._allowed_games import allowed_games
# _allowed_games, not _actions: _actions imports game_uninstall, which
# imports this module. The leaf module has the same function and no cycle.
from steam_backlog_enforcer._allowed_games import allowed_games
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
from steam_backlog_enforcer.config import State
logger = logging.getLogger(__name__)
@ -69,8 +71,8 @@ def _protected_name_stems() -> list[str]:
"""
try:
return [name for _, name in allowed_games(State.load()) if name]
# Catch-all on purpose: a best-effort safety net, not a hard dependency.
except Exception: # pylint: disable=broad-exception-caught
# Best effort, not a hard dependency: an unreadable state protects nothing.
except FAIL_SAFE_ERRORS:
logger.exception("Could not load allowed games for the deletion safety net")
return []

View File

@ -34,6 +34,7 @@ import os
import signal
from typing import TYPE_CHECKING, Final, Protocol
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
from steam_backlog_enforcer._leisure_never_kill import never_kill
from steam_backlog_enforcer._leisure_procs import (
exe_name,
@ -138,9 +139,8 @@ class LeisureKiller:
"""
try:
self._enforce(state, rules, verdict=verdict, stamp=stamp)
# Never raises: the enforce loop has no per-tick guard and must outlive
# any kill bug, so this catch-all is on purpose.
except Exception as exc: # pylint: disable=broad-exception-caught
# No per-tick guard upstream: outlive a failed kill; a bug stays loud.
except FAIL_SAFE_ERRORS as exc:
message = f"{type(exc).__name__}: {exc}"
if message != self._last_error:
self._last_error = message

View File

@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Final
from leisure_watch.facts import read_now
from leisure_watch.lockstate import paused_by
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
from steam_backlog_enforcer._playtime_state import _DELTA_CLAMP_FACTOR
if TYPE_CHECKING:
@ -120,8 +121,8 @@ def observe_shadow(owners: dict[int, str], *, stamp: float) -> ShadowVerdict:
if paused is not None:
return ShadowVerdict(paused_by=paused, keys=())
fact = read_now(NOW_PATH, stamp, max_age=_MAX_AGE_SECONDS)
# Catch-all on purpose: the loop must outlive any shadow input bug.
except Exception as exc: # pylint: disable=broad-exception-caught
# The loop must outlive any unreadable shadow input; a bug stays loud.
except FAIL_SAFE_ERRORS as exc:
if _should_warn(exc):
logger.warning(
"Shadow inputs unreadable (%r); failing closed.", exc, exc_info=True

View File

@ -22,6 +22,7 @@ from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from steam_backlog_enforcer import _web_auth, _web_process
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
from steam_backlog_enforcer._serve_stale import outdated_source
from steam_backlog_enforcer._web_errors import ApiError, not_found
from steam_backlog_enforcer._web_io import (
@ -101,8 +102,9 @@ class _Handler(BaseHTTPRequestHandler):
self._dispatch(method, split.path, _query(split.query))
except ApiError as exc:
send_error(self, exc)
# Catch-all on purpose: any bug answers 500, the server keeps serving.
except Exception: # pylint: disable=broad-exception-caught
# A failing route answers 500 and the server keeps serving; a bug
# outside the tuple goes to http.server's own error report.
except FAIL_SAFE_ERRORS:
logger.exception("%s %s failed", method, split.path)
msg = "The server failed to answer this request; see its log."
send_error(

View File

@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any, Final
from steam_backlog_enforcer._command_gate import lock_reason
from steam_backlog_enforcer._echo import routed_echo
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
from steam_backlog_enforcer._progress import use_progress
from steam_backlog_enforcer._prompter import PromptAbortedError, use_prompter
from steam_backlog_enforcer.config import Config, State, _atomic_write
@ -208,13 +209,17 @@ def run_job(job_dir: Path) -> int:
outcome = _failed(exc.message, EXIT_REFUSED)
except PrivilegedViaDaemonError as exc:
outcome = _failed(str(exc), EXIT_REFUSED)
# Catch-all on purpose: every job ends with a result, however it fails.
except Exception as exc: # pylint: disable=broad-exception-caught
except FAIL_SAFE_ERRORS as exc:
logger.exception("Job %s crashed", job_dir.name)
outcome = _failed(f"{type(exc).__name__}: {exc}")
# SystemExit from outside a handler, etc.: still write the result.
except BaseException as exc: # pylint: disable=broad-exception-caught
# SystemExit from outside a handler: still write the result.
except SystemExit as exc:
logger.exception("Job %s stopped", job_dir.name)
outcome = _failed(f"{type(exc).__name__}: {exc}")
# A bug: the job still ends with a result, then the process fails loudly.
except BaseException as exc:
logger.exception("Job %s crashed on a bug", job_dir.name)
_finish(job_dir, writer, _failed(f"{type(exc).__name__}: {exc}"))
raise
_finish(job_dir, writer, outcome)
return outcome.exit_code

View File

@ -2,10 +2,12 @@
from __future__ import annotations
from contextlib import contextmanager
import threading
from typing import TYPE_CHECKING
from unittest.mock import patch
from steam_backlog_enforcer import _web_process
from steam_backlog_enforcer import _web_process, _web_server
from steam_backlog_enforcer._web_process import _server
if TYPE_CHECKING:
@ -23,3 +25,19 @@ def clean_process_state() -> Iterator[None]:
yield
_web_process.RETIRING.clear()
_server.clear()
@contextmanager
def running_server() -> Iterator[_web_server.WebServer]:
"""Serve on an ephemeral loopback port in a thread."""
server = _web_server.create_server("127.0.0.1", 0)
thread = threading.Thread(
target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
)
thread.start()
try:
yield server
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2)

View File

@ -176,6 +176,21 @@ class TestHandle:
for _ in range(_ctl_server.MAX_CONNECTIONS):
assert srv.take_slot()
def test_a_bug_ends_the_thread_but_frees_the_slot(self) -> None:
"""A bug outside FAIL_SAFE_ERRORS escapes; the socket and slot do not leak."""
srv = _server()
conn = MagicMock()
assert srv.take_slot()
with (
patch.object(_ctl_server, "read_line", return_value=b"x"),
patch.object(srv, "dispatch", side_effect=AttributeError("bug")),
pytest.raises(AttributeError),
):
srv.handle(conn, 7)
conn.close.assert_called_once()
for _ in range(_ctl_server.MAX_CONNECTIONS):
assert srv.take_slot()
class TestDispatch:
"""_dispatch never raises."""
@ -197,6 +212,15 @@ class TestDispatch:
with patch.object(srv, "run", side_effect=RuntimeError("x")):
assert b'"error":"op_failed"' in srv.dispatch(b"", 0)
def test_a_programming_bug_is_loud(self) -> None:
"""An error outside FAIL_SAFE_ERRORS is not dressed up as op_failed."""
srv = _server()
with (
patch.object(srv, "run", side_effect=AttributeError("bug")),
pytest.raises(AttributeError),
):
srv.dispatch(b"", 0)
def test_run_rejects_unknown(self) -> None:
"""An unknown op raises CtlError."""
with pytest.raises(CtlError, match="unknown op"):

View File

@ -8,6 +8,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from unittest.mock import MagicMock, patch
import pytest
from steam_backlog_enforcer._game_names import (
_is_protected_name,
_normalize_game_name,
@ -69,6 +71,17 @@ class TestProtectedNameStems:
stems = _protected_name_stems()
assert stems == []
def test_a_programming_bug_is_loud(self) -> None:
"""An error outside FAIL_SAFE_ERRORS propagates."""
with (
patch(
"steam_backlog_enforcer.config.State.load",
side_effect=AttributeError("bug"),
),
pytest.raises(AttributeError, match="bug"),
):
_protected_name_stems()
class TestIsProtectedName:
"""Tests for _is_protected_name."""

View File

@ -229,3 +229,15 @@ class TestRunJob:
events = read_events(path)
assert [e["type"] for e in events[-2:]] == ["result", "state"]
assert events[-1]["state"] == state
def test_a_programming_bug_writes_a_result_then_raises(self) -> None:
"""An error outside FAIL_SAFE_ERRORS still ends the job, then is loud."""
path = make_job()
with (
patch(_RUN, side_effect=AttributeError("bug")),
pytest.raises(AttributeError, match="bug"),
):
_runner.run_job(path)
result = json.loads((path / RESULT_NAME).read_text(encoding="utf-8"))
assert (result["state"], result["summary"]) == ("failed", "AttributeError: bug")
assert read_events(path)[-1]["state"] == "failed"

View File

@ -183,6 +183,16 @@ class TestEnforce:
killer.enforce(_SPENT, _RULES, verdict=_verdict(None), stamp=stamp)
assert sum("Leisure kill failed" in r.message for r in caplog.records) == 1
def test_a_programming_bug_is_loud(self) -> None:
"""An error outside FAIL_SAFE_ERRORS propagates instead of being logged."""
killer = LeisureKiller(None)
with (
patch.object(_leisure_kill, "sweep_targets", side_effect=AttributeError),
pytest.raises(AttributeError),
):
for stamp in range(0, 34, 3):
killer.enforce(_SPENT, _RULES, verdict=_verdict(None), stamp=stamp)
def test_a_failed_sigkill_is_retried_and_logged_once_delivered(
self, proc: Path, kill: MagicMock
) -> None:

View File

@ -80,6 +80,16 @@ def test_a_successful_pause_check_still_pauses() -> None:
assert observe_shadow({}, stamp=T0) == ShadowVerdict("i3lock", ())
def test_a_programming_bug_is_loud() -> None:
"""An error outside FAIL_SAFE_ERRORS reaches the caller."""
with (
patch.object(_shadow, "paused_by", return_value=None),
patch.object(_shadow, "read_now", side_effect=AttributeError("bug")),
pytest.raises(AttributeError, match="bug"),
):
observe_shadow({}, stamp=T0)
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]

View File

@ -0,0 +1,56 @@
"""A route bug outside FAIL_SAFE_ERRORS is loud, and the server keeps serving."""
from __future__ import annotations
from http.client import HTTPConnection, RemoteDisconnected
from typing import TYPE_CHECKING
from unittest.mock import patch
import pytest
from steam_backlog_enforcer.tests._web_process_state import (
clean_process_state,
running_server,
)
if TYPE_CHECKING:
from collections.abc import Iterator
from steam_backlog_enforcer._web_io import Reply, Request
_PKG = "steam_backlog_enforcer._web_server"
@pytest.fixture(autouse=True)
def _clean_process_state() -> Iterator[None]:
"""No retiring flag, and never "stale"."""
yield from clean_process_state()
def _get(port: int, path: str) -> int:
"""One GET; the status, or ``-1`` if the server hung up without one."""
conn = HTTPConnection("127.0.0.1", port, timeout=5)
try:
conn.request("GET", path)
return conn.getresponse().status
except RemoteDisconnected:
return -1
finally:
conn.close()
def test_a_bug_drops_the_request_and_the_server_survives() -> None:
"""The buggy request gets no answer; the next request is served."""
def route(_request: Request) -> Reply:
msg = "bug"
raise AttributeError(msg)
with running_server() as server:
with (
patch(f"{_PKG}.resolve", return_value=(route, ())),
patch.object(server, "handle_error") as handle_error,
):
assert _get(server.port, "/api/budget") == -1
handle_error.assert_called_once()
assert _get(server.port, "/api/jobs") == 200

View File

@ -2,10 +2,8 @@
from __future__ import annotations
from contextlib import contextmanager
from http.client import HTTPConnection
import json
import threading
from typing import TYPE_CHECKING
from unittest.mock import MagicMock, patch
@ -16,7 +14,10 @@ from steam_backlog_enforcer._web_io import Reply, Request, ok
from steam_backlog_enforcer._web_server import _STALE_MSG
from steam_backlog_enforcer.jobs._daemon_jobs import DaemonOutcome, record_daemon_job
from steam_backlog_enforcer.jobs._events import now_iso
from steam_backlog_enforcer.tests._web_process_state import clean_process_state
from steam_backlog_enforcer.tests._web_process_state import (
clean_process_state,
running_server,
)
if TYPE_CHECKING:
from collections.abc import Iterator
@ -31,20 +32,7 @@ def _clean_process_state() -> Iterator[None]:
yield from clean_process_state()
@contextmanager
def _running() -> Iterator[_web_server.WebServer]:
"""Serve on an ephemeral loopback port in a thread."""
server = _web_server.create_server("127.0.0.1", 0)
thread = threading.Thread(
target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
)
thread.start()
try:
yield server
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2)
_running = running_server
def _auth(server: _web_server.WebServer) -> dict[str, str]: