mirror of
https://github.com/kuhyx/steam-backlog-enforcer.git
synced 2026-11-22 09:42:18 +01:00
Narrow the live enforcer's catch-all handlers
Every except Exception in the daemon, ctl socket, web server and job runner now catches FAIL_SAFE_ERRORS (OSError, ValueError, TypeError, LookupError, RuntimeError, SubprocessError). A bug outside it is loud: the ctl thread dies with its slot freed, http.server reports it, a job writes its failed result and re-raises. Todo: 9b9342ba Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
db44ca5b0b
commit
444f619e6d
@ -41,6 +41,7 @@ from steam_backlog_enforcer._ctl_wire import (
|
||||
read_line,
|
||||
socket_is_live,
|
||||
)
|
||||
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
@ -172,8 +173,8 @@ class CtlServer:
|
||||
line = read_line(conn)
|
||||
if line is not None:
|
||||
self._write(conn, self.dispatch(line, uid))
|
||||
# Catch-all on purpose: a dying connection must not kill its thread.
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# A dying connection (timeout, hang-up) must not kill its thread.
|
||||
except OSError:
|
||||
logger.debug("Control connection dropped", exc_info=True)
|
||||
finally:
|
||||
conn.close()
|
||||
@ -186,14 +187,15 @@ class CtlServer:
|
||||
conn.sendall(payload)
|
||||
|
||||
def dispatch(self, line: bytes, uid: int) -> bytes:
|
||||
"""Turn a request line into a response line. Never raises."""
|
||||
"""Turn a request line into a response line; op failures become errors."""
|
||||
try:
|
||||
return ok_response(self.run(line, uid))
|
||||
except CtlError as exc:
|
||||
logger.info("Control socket: refused (%s)", exc.code)
|
||||
return error_response(exc)
|
||||
# Catch-all on purpose: any op bug answers "internal error", never raises.
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# A failing op answers "internal error"; a bug outside the tuple ends
|
||||
# only this connection's thread (``handle`` still frees its slot).
|
||||
except FAIL_SAFE_ERRORS:
|
||||
logger.exception("Control socket op crashed")
|
||||
return error_response(CtlError(OP_FAILED, "internal error"))
|
||||
|
||||
|
||||
30
steam_backlog_enforcer/_fail_safe.py
Normal file
30
steam_backlog_enforcer/_fail_safe.py
Normal file
@ -0,0 +1,30 @@
|
||||
"""The exceptions a fail-safe handler absorbs; anything else propagates.
|
||||
|
||||
No handler in the live enforcer catches ``Exception`` (kuhy's call, todo
|
||||
9b9342ba). A handler that must outlive a failing input or op catches this
|
||||
tuple instead:
|
||||
|
||||
* ``OSError``: files, sockets, ``/proc``, signals (incl. timeouts, a peer
|
||||
that hung up, ``ProcessLookupError``);
|
||||
* ``ValueError``: malformed data, incl. ``json`` and Unicode decode errors;
|
||||
* ``TypeError`` and ``LookupError``: a payload of the wrong shape;
|
||||
* ``RuntimeError``: incl. ``RecursionError``, which leisure_watch has raised;
|
||||
* ``subprocess.SubprocessError``: a helper that failed or timed out.
|
||||
|
||||
A programming bug outside it (``AttributeError``, ``NameError``, ...) is loud:
|
||||
it reaches the thread's or the process's own error reporting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from typing import Final
|
||||
|
||||
FAIL_SAFE_ERRORS: Final[tuple[type[Exception], ...]] = (
|
||||
OSError,
|
||||
ValueError,
|
||||
TypeError,
|
||||
LookupError,
|
||||
RuntimeError,
|
||||
subprocess.SubprocessError,
|
||||
)
|
||||
@ -11,9 +11,11 @@ import difflib
|
||||
import logging
|
||||
import re
|
||||
|
||||
from steam_backlog_enforcer._allowed_games import allowed_games
|
||||
|
||||
# _allowed_games, not _actions: _actions imports game_uninstall, which
|
||||
# imports this module. The leaf module has the same function and no cycle.
|
||||
from steam_backlog_enforcer._allowed_games import allowed_games
|
||||
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
|
||||
from steam_backlog_enforcer.config import State
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -69,8 +71,8 @@ def _protected_name_stems() -> list[str]:
|
||||
"""
|
||||
try:
|
||||
return [name for _, name in allowed_games(State.load()) if name]
|
||||
# Catch-all on purpose: a best-effort safety net, not a hard dependency.
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# Best effort, not a hard dependency: an unreadable state protects nothing.
|
||||
except FAIL_SAFE_ERRORS:
|
||||
logger.exception("Could not load allowed games for the deletion safety net")
|
||||
return []
|
||||
|
||||
|
||||
@ -34,6 +34,7 @@ import os
|
||||
import signal
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
|
||||
from steam_backlog_enforcer._leisure_never_kill import never_kill
|
||||
from steam_backlog_enforcer._leisure_procs import (
|
||||
exe_name,
|
||||
@ -138,9 +139,8 @@ class LeisureKiller:
|
||||
"""
|
||||
try:
|
||||
self._enforce(state, rules, verdict=verdict, stamp=stamp)
|
||||
# Never raises: the enforce loop has no per-tick guard and must outlive
|
||||
# any kill bug, so this catch-all is on purpose.
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
# No per-tick guard upstream: outlive a failed kill; a bug stays loud.
|
||||
except FAIL_SAFE_ERRORS as exc:
|
||||
message = f"{type(exc).__name__}: {exc}"
|
||||
if message != self._last_error:
|
||||
self._last_error = message
|
||||
|
||||
@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Final
|
||||
from leisure_watch.facts import read_now
|
||||
from leisure_watch.lockstate import paused_by
|
||||
|
||||
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
|
||||
from steam_backlog_enforcer._playtime_state import _DELTA_CLAMP_FACTOR
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@ -120,8 +121,8 @@ def observe_shadow(owners: dict[int, str], *, stamp: float) -> ShadowVerdict:
|
||||
if paused is not None:
|
||||
return ShadowVerdict(paused_by=paused, keys=())
|
||||
fact = read_now(NOW_PATH, stamp, max_age=_MAX_AGE_SECONDS)
|
||||
# Catch-all on purpose: the loop must outlive any shadow input bug.
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
# The loop must outlive any unreadable shadow input; a bug stays loud.
|
||||
except FAIL_SAFE_ERRORS as exc:
|
||||
if _should_warn(exc):
|
||||
logger.warning(
|
||||
"Shadow inputs unreadable (%r); failing closed.", exc, exc_info=True
|
||||
|
||||
@ -22,6 +22,7 @@ from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from steam_backlog_enforcer import _web_auth, _web_process
|
||||
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
|
||||
from steam_backlog_enforcer._serve_stale import outdated_source
|
||||
from steam_backlog_enforcer._web_errors import ApiError, not_found
|
||||
from steam_backlog_enforcer._web_io import (
|
||||
@ -101,8 +102,9 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
self._dispatch(method, split.path, _query(split.query))
|
||||
except ApiError as exc:
|
||||
send_error(self, exc)
|
||||
# Catch-all on purpose: any bug answers 500, the server keeps serving.
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# A failing route answers 500 and the server keeps serving; a bug
|
||||
# outside the tuple goes to http.server's own error report.
|
||||
except FAIL_SAFE_ERRORS:
|
||||
logger.exception("%s %s failed", method, split.path)
|
||||
msg = "The server failed to answer this request; see its log."
|
||||
send_error(
|
||||
|
||||
@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from steam_backlog_enforcer._command_gate import lock_reason
|
||||
from steam_backlog_enforcer._echo import routed_echo
|
||||
from steam_backlog_enforcer._fail_safe import FAIL_SAFE_ERRORS
|
||||
from steam_backlog_enforcer._progress import use_progress
|
||||
from steam_backlog_enforcer._prompter import PromptAbortedError, use_prompter
|
||||
from steam_backlog_enforcer.config import Config, State, _atomic_write
|
||||
@ -208,13 +209,17 @@ def run_job(job_dir: Path) -> int:
|
||||
outcome = _failed(exc.message, EXIT_REFUSED)
|
||||
except PrivilegedViaDaemonError as exc:
|
||||
outcome = _failed(str(exc), EXIT_REFUSED)
|
||||
# Catch-all on purpose: every job ends with a result, however it fails.
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
except FAIL_SAFE_ERRORS as exc:
|
||||
logger.exception("Job %s crashed", job_dir.name)
|
||||
outcome = _failed(f"{type(exc).__name__}: {exc}")
|
||||
# SystemExit from outside a handler, etc.: still write the result.
|
||||
except BaseException as exc: # pylint: disable=broad-exception-caught
|
||||
# SystemExit from outside a handler: still write the result.
|
||||
except SystemExit as exc:
|
||||
logger.exception("Job %s stopped", job_dir.name)
|
||||
outcome = _failed(f"{type(exc).__name__}: {exc}")
|
||||
# A bug: the job still ends with a result, then the process fails loudly.
|
||||
except BaseException as exc:
|
||||
logger.exception("Job %s crashed on a bug", job_dir.name)
|
||||
_finish(job_dir, writer, _failed(f"{type(exc).__name__}: {exc}"))
|
||||
raise
|
||||
_finish(job_dir, writer, outcome)
|
||||
return outcome.exit_code
|
||||
|
||||
@ -2,10 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import patch
|
||||
|
||||
from steam_backlog_enforcer import _web_process
|
||||
from steam_backlog_enforcer import _web_process, _web_server
|
||||
from steam_backlog_enforcer._web_process import _server
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@ -23,3 +25,19 @@ def clean_process_state() -> Iterator[None]:
|
||||
yield
|
||||
_web_process.RETIRING.clear()
|
||||
_server.clear()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def running_server() -> Iterator[_web_server.WebServer]:
|
||||
"""Serve on an ephemeral loopback port in a thread."""
|
||||
server = _web_server.create_server("127.0.0.1", 0)
|
||||
thread = threading.Thread(
|
||||
target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
|
||||
)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=2)
|
||||
|
||||
@ -176,6 +176,21 @@ class TestHandle:
|
||||
for _ in range(_ctl_server.MAX_CONNECTIONS):
|
||||
assert srv.take_slot()
|
||||
|
||||
def test_a_bug_ends_the_thread_but_frees_the_slot(self) -> None:
|
||||
"""A bug outside FAIL_SAFE_ERRORS escapes; the socket and slot do not leak."""
|
||||
srv = _server()
|
||||
conn = MagicMock()
|
||||
assert srv.take_slot()
|
||||
with (
|
||||
patch.object(_ctl_server, "read_line", return_value=b"x"),
|
||||
patch.object(srv, "dispatch", side_effect=AttributeError("bug")),
|
||||
pytest.raises(AttributeError),
|
||||
):
|
||||
srv.handle(conn, 7)
|
||||
conn.close.assert_called_once()
|
||||
for _ in range(_ctl_server.MAX_CONNECTIONS):
|
||||
assert srv.take_slot()
|
||||
|
||||
|
||||
class TestDispatch:
|
||||
"""_dispatch never raises."""
|
||||
@ -197,6 +212,15 @@ class TestDispatch:
|
||||
with patch.object(srv, "run", side_effect=RuntimeError("x")):
|
||||
assert b'"error":"op_failed"' in srv.dispatch(b"", 0)
|
||||
|
||||
def test_a_programming_bug_is_loud(self) -> None:
|
||||
"""An error outside FAIL_SAFE_ERRORS is not dressed up as op_failed."""
|
||||
srv = _server()
|
||||
with (
|
||||
patch.object(srv, "run", side_effect=AttributeError("bug")),
|
||||
pytest.raises(AttributeError),
|
||||
):
|
||||
srv.dispatch(b"", 0)
|
||||
|
||||
def test_run_rejects_unknown(self) -> None:
|
||||
"""An unknown op raises CtlError."""
|
||||
with pytest.raises(CtlError, match="unknown op"):
|
||||
|
||||
@ -8,6 +8,8 @@ from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer._game_names import (
|
||||
_is_protected_name,
|
||||
_normalize_game_name,
|
||||
@ -69,6 +71,17 @@ class TestProtectedNameStems:
|
||||
stems = _protected_name_stems()
|
||||
assert stems == []
|
||||
|
||||
def test_a_programming_bug_is_loud(self) -> None:
|
||||
"""An error outside FAIL_SAFE_ERRORS propagates."""
|
||||
with (
|
||||
patch(
|
||||
"steam_backlog_enforcer.config.State.load",
|
||||
side_effect=AttributeError("bug"),
|
||||
),
|
||||
pytest.raises(AttributeError, match="bug"),
|
||||
):
|
||||
_protected_name_stems()
|
||||
|
||||
|
||||
class TestIsProtectedName:
|
||||
"""Tests for _is_protected_name."""
|
||||
|
||||
@ -229,3 +229,15 @@ class TestRunJob:
|
||||
events = read_events(path)
|
||||
assert [e["type"] for e in events[-2:]] == ["result", "state"]
|
||||
assert events[-1]["state"] == state
|
||||
|
||||
def test_a_programming_bug_writes_a_result_then_raises(self) -> None:
|
||||
"""An error outside FAIL_SAFE_ERRORS still ends the job, then is loud."""
|
||||
path = make_job()
|
||||
with (
|
||||
patch(_RUN, side_effect=AttributeError("bug")),
|
||||
pytest.raises(AttributeError, match="bug"),
|
||||
):
|
||||
_runner.run_job(path)
|
||||
result = json.loads((path / RESULT_NAME).read_text(encoding="utf-8"))
|
||||
assert (result["state"], result["summary"]) == ("failed", "AttributeError: bug")
|
||||
assert read_events(path)[-1]["state"] == "failed"
|
||||
|
||||
@ -183,6 +183,16 @@ class TestEnforce:
|
||||
killer.enforce(_SPENT, _RULES, verdict=_verdict(None), stamp=stamp)
|
||||
assert sum("Leisure kill failed" in r.message for r in caplog.records) == 1
|
||||
|
||||
def test_a_programming_bug_is_loud(self) -> None:
|
||||
"""An error outside FAIL_SAFE_ERRORS propagates instead of being logged."""
|
||||
killer = LeisureKiller(None)
|
||||
with (
|
||||
patch.object(_leisure_kill, "sweep_targets", side_effect=AttributeError),
|
||||
pytest.raises(AttributeError),
|
||||
):
|
||||
for stamp in range(0, 34, 3):
|
||||
killer.enforce(_SPENT, _RULES, verdict=_verdict(None), stamp=stamp)
|
||||
|
||||
def test_a_failed_sigkill_is_retried_and_logged_once_delivered(
|
||||
self, proc: Path, kill: MagicMock
|
||||
) -> None:
|
||||
|
||||
@ -80,6 +80,16 @@ def test_a_successful_pause_check_still_pauses() -> None:
|
||||
assert observe_shadow({}, stamp=T0) == ShadowVerdict("i3lock", ())
|
||||
|
||||
|
||||
def test_a_programming_bug_is_loud() -> None:
|
||||
"""An error outside FAIL_SAFE_ERRORS reaches the caller."""
|
||||
with (
|
||||
patch.object(_shadow, "paused_by", return_value=None),
|
||||
patch.object(_shadow, "read_now", side_effect=AttributeError("bug")),
|
||||
pytest.raises(AttributeError, match="bug"),
|
||||
):
|
||||
observe_shadow({}, stamp=T0)
|
||||
|
||||
|
||||
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
|
||||
return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
|
||||
|
||||
|
||||
56
steam_backlog_enforcer/tests/test_web_server_fail_safe.py
Normal file
56
steam_backlog_enforcer/tests/test_web_server_fail_safe.py
Normal file
@ -0,0 +1,56 @@
|
||||
"""A route bug outside FAIL_SAFE_ERRORS is loud, and the server keeps serving."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from http.client import HTTPConnection, RemoteDisconnected
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer.tests._web_process_state import (
|
||||
clean_process_state,
|
||||
running_server,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
from steam_backlog_enforcer._web_io import Reply, Request
|
||||
|
||||
_PKG = "steam_backlog_enforcer._web_server"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_process_state() -> Iterator[None]:
|
||||
"""No retiring flag, and never "stale"."""
|
||||
yield from clean_process_state()
|
||||
|
||||
|
||||
def _get(port: int, path: str) -> int:
|
||||
"""One GET; the status, or ``-1`` if the server hung up without one."""
|
||||
conn = HTTPConnection("127.0.0.1", port, timeout=5)
|
||||
try:
|
||||
conn.request("GET", path)
|
||||
return conn.getresponse().status
|
||||
except RemoteDisconnected:
|
||||
return -1
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_a_bug_drops_the_request_and_the_server_survives() -> None:
|
||||
"""The buggy request gets no answer; the next request is served."""
|
||||
|
||||
def route(_request: Request) -> Reply:
|
||||
msg = "bug"
|
||||
raise AttributeError(msg)
|
||||
|
||||
with running_server() as server:
|
||||
with (
|
||||
patch(f"{_PKG}.resolve", return_value=(route, ())),
|
||||
patch.object(server, "handle_error") as handle_error,
|
||||
):
|
||||
assert _get(server.port, "/api/budget") == -1
|
||||
handle_error.assert_called_once()
|
||||
assert _get(server.port, "/api/jobs") == 200
|
||||
@ -2,10 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from http.client import HTTPConnection
|
||||
import json
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@ -16,7 +14,10 @@ from steam_backlog_enforcer._web_io import Reply, Request, ok
|
||||
from steam_backlog_enforcer._web_server import _STALE_MSG
|
||||
from steam_backlog_enforcer.jobs._daemon_jobs import DaemonOutcome, record_daemon_job
|
||||
from steam_backlog_enforcer.jobs._events import now_iso
|
||||
from steam_backlog_enforcer.tests._web_process_state import clean_process_state
|
||||
from steam_backlog_enforcer.tests._web_process_state import (
|
||||
clean_process_state,
|
||||
running_server,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
@ -31,20 +32,7 @@ def _clean_process_state() -> Iterator[None]:
|
||||
yield from clean_process_state()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _running() -> Iterator[_web_server.WebServer]:
|
||||
"""Serve on an ephemeral loopback port in a thread."""
|
||||
server = _web_server.create_server("127.0.0.1", 0)
|
||||
thread = threading.Thread(
|
||||
target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
|
||||
)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=2)
|
||||
_running = running_server
|
||||
|
||||
|
||||
def _auth(server: _web_server.WebServer) -> dict[str, str]:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user