From 57d262b71161fcefd73e816dfeaf79599ef641d5 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Tue, 10 Nov 2026 16:57:35 +0100 Subject: [PATCH] Pylint gate 10 over the whole package Raise the pylint hook to --fail-under=10.0 and make it lint the whole package on every run (pass_filenames: false): over the staged subset a 9.99 tree passed at commit time. Ports de2aea3 (9b9342ba-1) onto main, minus its web/ and requirements.txt bumps and the iptables dedupe f1822f1 already landed; _hltb_confidence and _hltb_search keep main's 429 handling. _search_batch splits out _acquire_auth / _search_headers, and _stats.py is back to 250 lines. Drops the min-similarity-lines = 8 floor (0 duplicate-code hits at the default) and sets max-attributes = 15 for the flat persisted records, as kuhy decided. Tests stay outside pylint, as at every site already at 10. Todo: fda71ed1 Co-Authored-By: Claude Opus 5.5 --- .pre-commit-config.yaml | 10 +- pyproject.toml | 11 +-- steam_backlog_enforcer/_ctl_server.py | 15 ++- steam_backlog_enforcer/_fast_install.py | 24 +++-- .../_fast_install_progress.py | 18 ++-- steam_backlog_enforcer/_game_names.py | 1 + steam_backlog_enforcer/_hltb_comp_extract.py | 22 +++++ steam_backlog_enforcer/_hltb_confidence.py | 33 ++++--- steam_backlog_enforcer/_hltb_detail.py | 96 ++++++++----------- steam_backlog_enforcer/_hltb_matching.py | 8 +- steam_backlog_enforcer/_hltb_search.py | 74 ++++++++------ steam_backlog_enforcer/_hltb_types.py | 2 +- steam_backlog_enforcer/_leisure_kill.py | 12 ++- steam_backlog_enforcer/_shadow.py | 3 +- steam_backlog_enforcer/_stats.py | 85 ++++++++-------- steam_backlog_enforcer/_stats_gathering.py | 94 ++++++++++-------- steam_backlog_enforcer/_steam_models.py | 5 +- steam_backlog_enforcer/_web_routes.py | 2 +- steam_backlog_enforcer/_web_server.py | 3 +- steam_backlog_enforcer/_web_sse.py | 6 +- steam_backlog_enforcer/jobs/_runner.py | 13 ++- steam_backlog_enforcer/jobs/_spawn.py | 32 ++++--- steam_backlog_enforcer/library_hider.py | 4 +- steam_backlog_enforcer/scanning.py | 17 ++-- steam_backlog_enforcer/store_blocker.py | 5 +- 25 files changed, 336 insertions(+), 259 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09ab164..1083ed6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -124,13 +124,19 @@ repos: rev: v4.0.9 hooks: - id: pylint - exclude: ^scripts/file_length/ + # Whole tree on every run, never the staged subset: `--fail-under` + # grades a score, and a score over whatever happened to be staged let a + # 9.99 tree pass at commit time and fail in CI. scripts/file_length/ + # is the shared copy from ~/src/utils, linted there. + pass_filenames: false args: - --rcfile=pyproject.toml - - --fail-under=8.0 + - --fail-under=10.0 # Serial: parallel pylint pickles PyLinter with dill, which crashes on # Python 3.15 (removed co_lnotab). Same reason as jobs = 1 in pyproject. - --jobs=1 + - steam_backlog_enforcer + - scripts/check_requirements_sync.py # pylint resolves imports in its OWN isolated env, so anything the # package imports has to be listed here or it reports import-error and # drags the score under --fail-under. `mcp` is a real runtime import diff --git a/pyproject.toml b/pyproject.toml index f58c19c..7d7a187 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -169,15 +169,14 @@ disable = [ "use-implicit-booleaness-not-comparison-to-zero", ] -[tool.pylint.similarities] -# Same duplicate-code threshold as diet-guard: shorter matches are mostly -# import blocks and subprocess.run keyword lists, not real copy-paste. -min-similarity-lines = 8 - [tool.pylint.design] min-public-methods = 0 max-module-lines = 250 -max-attributes = 10 +# 15 = the widest persisted record (config.Config). Config, State and +# PlaytimeState are flat JSON files the live daemon reads and writes; WebGame +# is the UI's row shape. Splitting them under 10 would change the on-disk shape +# or hide it behind property forwarders written only for this check. +max-attributes = 15 [tool.pylint.typecheck] generated-members = [ diff --git a/steam_backlog_enforcer/_ctl_server.py b/steam_backlog_enforcer/_ctl_server.py index 6006904..01fad02 100644 --- a/steam_backlog_enforcer/_ctl_server.py +++ b/steam_backlog_enforcer/_ctl_server.py @@ -151,7 +151,7 @@ class CtlServer: logger.warning("Control socket: refused a connection from uid %d", uid) conn.close() return - if not self._slots.acquire(blocking=False): + if not self._take_slot(): logger.warning("Control socket: too many connections, dropping one") conn.close() return @@ -159,13 +159,21 @@ class CtlServer: target=self._handle, args=(conn, uid), name="ctl-conn", daemon=True ).start() + def _take_slot(self) -> bool: + """Claim a connection slot without blocking; ``_handle`` releases it. + + No ``with``: the slot is released by the worker thread, not here. + """ + return self._slots.acquire(blocking=False) + def _handle(self, conn: socket.socket, uid: int) -> None: """Serve one connection, always releasing its slot and socket.""" try: line = read_line(conn) if line is not None: self._write(conn, self._dispatch(line, uid)) - except Exception: + # Catch-all on purpose: a dying connection must not kill its thread. + except Exception: # pylint: disable=broad-exception-caught logger.debug("Control connection dropped", exc_info=True) finally: conn.close() @@ -184,7 +192,8 @@ class CtlServer: except CtlError as exc: logger.info("Control socket: refused (%s)", exc.code) return error_response(exc) - except Exception: + # Catch-all on purpose: any op bug answers "internal error", never raises. + except Exception: # pylint: disable=broad-exception-caught logger.exception("Control socket op crashed") return error_response(CtlError(OP_FAILED, "internal error")) diff --git a/steam_backlog_enforcer/_fast_install.py b/steam_backlog_enforcer/_fast_install.py index 809ca9a..5927404 100644 --- a/steam_backlog_enforcer/_fast_install.py +++ b/steam_backlog_enforcer/_fast_install.py @@ -46,6 +46,7 @@ from steam_backlog_enforcer._steam_state import is_game_fully_installed if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path + from typing import BinaryIO logger = logging.getLogger(__name__) @@ -134,6 +135,21 @@ def poll_fast_installs() -> None: _fip.poll_trackers({run.app_id for run in _RUNS}, is_game_fully_installed) +def _start_session(cmd: list[str], log: BinaryIO) -> subprocess.Popen[bytes]: + """Start *cmd* in its own session writing to *log*; the caller owns it. + + No ``with``: the run outlives this call and is reaped by + :func:`poll_fast_installs`. + """ + return subprocess.Popen( + cmd, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + def _spawn_detached(cmd: list[str], app_id: int, label: str = "") -> None: """Start the installer in its own session, output to a log file.""" # Per user: the daemon's log is root-owned, and a web job runs as the @@ -149,13 +165,7 @@ def _spawn_detached(cmd: list[str], app_id: int, label: str = "") -> None: with log_path.open("ab") as log: # Append mode starts at the end: earlier runs' lines are never parsed. _fip.track(app_id, label, log_path, offset=log.tell(), now=time.monotonic()) - proc = subprocess.Popen( - cmd, - stdin=subprocess.DEVNULL, - stdout=log, - stderr=subprocess.STDOUT, - start_new_session=True, - ) + proc = _start_session(cmd, log) _RUNS.append(_DetachedRun(app_id, proc, log_path)) logger.info("Started steam-game-installer for AppID=%d (log: %s)", app_id, log_path) diff --git a/steam_backlog_enforcer/_fast_install_progress.py b/steam_backlog_enforcer/_fast_install_progress.py index 055a66e..15f7202 100644 --- a/steam_backlog_enforcer/_fast_install_progress.py +++ b/steam_backlog_enforcer/_fast_install_progress.py @@ -31,6 +31,7 @@ the daemon's own enforce loop polls the trackers. from __future__ import annotations +import contextlib from dataclasses import dataclass import re import threading @@ -109,16 +110,13 @@ class InstallTracker: A log that is missing or unreadable changes nothing: the run itself is judged by its exit status, never by its log. """ - try: - with self._path.open("rb") as log: - log.seek(self._offset) - while chunk := log.read(_READ_CHUNK): - self._offset += len(chunk) - *lines, self._partial = (self._partial + chunk).split(b"\n") - for line in lines: - self._feed(line.decode("utf-8", "replace").rstrip("\r")) - except OSError: - return + with contextlib.suppress(OSError), self._path.open("rb") as log: + log.seek(self._offset) + while chunk := log.read(_READ_CHUNK): + self._offset += len(chunk) + *lines, self._partial = (self._partial + chunk).split(b"\n") + for line in lines: + self._feed(line.decode("utf-8", "replace").rstrip("\r")) def _feed(self, line: str) -> None: """Apply one log line to the phase / percent state.""" diff --git a/steam_backlog_enforcer/_game_names.py b/steam_backlog_enforcer/_game_names.py index a67bcf3..9a322a7 100644 --- a/steam_backlog_enforcer/_game_names.py +++ b/steam_backlog_enforcer/_game_names.py @@ -69,6 +69,7 @@ def _protected_name_stems() -> list[str]: """ try: return [name for _, name in allowed_games(State.load()) if name] + # Catch-all on purpose: a best-effort safety net, not a hard dependency. except Exception: # pylint: disable=broad-exception-caught logger.exception("Could not load allowed games for the deletion safety net") return [] diff --git a/steam_backlog_enforcer/_hltb_comp_extract.py b/steam_backlog_enforcer/_hltb_comp_extract.py index 7b4f82e..735f45c 100644 --- a/steam_backlog_enforcer/_hltb_comp_extract.py +++ b/steam_backlog_enforcer/_hltb_comp_extract.py @@ -31,6 +31,8 @@ from steam_backlog_enforcer._hltb_types import ( if TYPE_CHECKING: import asyncio + from steam_backlog_enforcer._hltb_types import HLTBResult + logger = logging.getLogger(__name__) _NEXT_DATA_RE = re.compile( @@ -130,3 +132,23 @@ async def _fetch_detail_one( ) record_failure(f"detail {reason}") return None + + +def _collect_dlc_relationships( + valid: list[HLTBResult], + details: list[dict[str, Any] | None], +) -> tuple[dict[int, list[tuple[int, float]]], list[int]]: + """Collect DLC relationship IDs for all base-game detail responses.""" + by_app: dict[int, list[tuple[int, float]]] = {} + unique_dlc_ids: set[int] = set() + + for result, game_data in zip(valid, details, strict=False): + if game_data is None: + continue + dlc_rels = _extract_dlc_relationships(game_data) + by_app[result.app_id] = dlc_rels + for dlc_id, _fallback_hours in dlc_rels: + if dlc_id > 0: + unique_dlc_ids.add(dlc_id) + + return by_app, sorted(unique_dlc_ids) diff --git a/steam_backlog_enforcer/_hltb_confidence.py b/steam_backlog_enforcer/_hltb_confidence.py index 077a70e..87f27cc 100644 --- a/steam_backlog_enforcer/_hltb_confidence.py +++ b/steam_backlog_enforcer/_hltb_confidence.py @@ -120,6 +120,23 @@ def fetch_hltb_confidence_cached( return cache +def _log_detail_plan(n_rush: int, n_id: int) -> None: + """Log what a detail fetch is about to do.""" + if n_rush and n_id: + logger.info( + "Fetching HLTB detail for %d games missing rush/leisure data" + " + %d games missing game ID...", + n_rush, + n_id, + ) + elif n_rush: + logger.info( + "Fetching HLTB detail for %d games missing rush/leisure data...", n_rush + ) + else: + logger.info("Backfilling HLTB game ID for %d game(s)...", n_id) + + def fetch_hltb_detail_missing( games: list[tuple[int, str]], progress_cb: ProgressCb | None = None, @@ -167,21 +184,7 @@ def fetch_hltb_detail_missing( prior_hours = {aid: cache[aid] for aid, _ in missing if aid in cache} - n_rush = len(missing_rush) - n_id = len(missing_id_only) - if n_rush and n_id: - logger.info( - "Fetching HLTB detail for %d games missing rush/leisure data" - " + %d games missing game ID...", - n_rush, - n_id, - ) - elif n_rush: - logger.info( - "Fetching HLTB detail for %d games missing rush/leisure data...", n_rush - ) - else: - logger.info("Backfilling HLTB game ID for %d game(s)...", n_id) + _log_detail_plan(len(missing_rush), len(missing_id_only)) report = reporting_progress_cb("Fetching HLTB details", len(missing), progress_cb) try: elapsed = fetch_hltb_times_timed(missing, cache, polls, report, extras) diff --git a/steam_backlog_enforcer/_hltb_detail.py b/steam_backlog_enforcer/_hltb_detail.py index bbfdf3d..f8f1253 100644 --- a/steam_backlog_enforcer/_hltb_detail.py +++ b/steam_backlog_enforcer/_hltb_detail.py @@ -9,6 +9,7 @@ from typing import Any import aiohttp from steam_backlog_enforcer._hltb_comp_extract import ( + _collect_dlc_relationships, _extract_comp_100_avg_and_high, _extract_dlc_relationships, _fetch_detail_one, @@ -141,6 +142,27 @@ def _apply_detail_to_extras( return leisure +async def _fetch_all_details( + valid: list[HLTBResult], +) -> tuple[ + list[dict[str, Any] | None], dict[int, list[tuple[int, float]]], dict[int, float] +]: + """Fetch every detail page, then the leisure hours of every DLC they list.""" + timeout = aiohttp.ClientTimeout(total=30, sock_read=20) + sem = asyncio.Semaphore(MAX_CONCURRENT) + connector = aiohttp.TCPConnector(limit=MAX_CONCURRENT, keepalive_timeout=30) + async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session: + coros = [_fetch_detail_one(sem, session, r.hltb_game_id) for r in valid] + names = [r.game_name for r in valid] + details = await gather_tracked("Fetching HLTB detail pages", coros, names) + dlc_relationships_by_app, dlc_ids = _collect_dlc_relationships(valid, details) + return ( + details, + dlc_relationships_by_app, + await _fetch_dlc_leisure_hours(sem, session, dlc_ids), + ) + + async def _fetch_leisure_times( search_results: list[HLTBResult], cache: dict[int, float], @@ -163,65 +185,27 @@ async def _fetch_leisure_times( if not valid: return - timeout = aiohttp.ClientTimeout(total=30, sock_read=20) - sem = asyncio.Semaphore(MAX_CONCURRENT) - connector = aiohttp.TCPConnector( - limit=MAX_CONCURRENT, - keepalive_timeout=30, - ) - - total = len(valid) - done = 0 + details, dlc_relationships_by_app, dlc_hours_by_id = await _fetch_all_details(valid) found = 0 + for done, (r, game_data) in enumerate(zip(valid, details, strict=False), 1): + if game_data is not None: + leisure = _apply_detail_to_extras( + r.app_id, + game_data, + dlc_relationships_by_app.get(r.app_id, []), + dlc_hours_by_id, + extras, + ) + if leisure > 0: + r.completionist_hours = leisure + cache[r.app_id] = leisure + found += 1 - async with aiohttp.ClientSession( - timeout=timeout, - connector=connector, - ) as session: - coros = [_fetch_detail_one(sem, session, r.hltb_game_id) for r in valid] - names = [r.game_name for r in valid] - details = await gather_tracked("Fetching HLTB detail pages", coros, names) + if progress_cb is not None: + progress_cb(done, len(valid), found, r.game_name) - dlc_relationships_by_app, dlc_ids = _collect_dlc_relationships(valid, details) - dlc_hours_by_id = await _fetch_dlc_leisure_hours(sem, session, dlc_ids) - - for r, game_data in zip(valid, details, strict=False): - done += 1 - if game_data is not None: - dlc_rels = dlc_relationships_by_app.get(r.app_id, []) - leisure = _apply_detail_to_extras( - r.app_id, game_data, dlc_rels, dlc_hours_by_id, extras - ) - if leisure > 0: - r.completionist_hours = leisure - cache[r.app_id] = leisure - found += 1 - - if progress_cb is not None: - progress_cb(done, total, found, r.game_name) - - if not done % _SAVE_INTERVAL: - save_hltb_cache(cache, polls, extras) - - -def _collect_dlc_relationships( - valid: list[HLTBResult], - details: list[dict[str, Any] | None], -) -> tuple[dict[int, list[tuple[int, float]]], list[int]]: - """Collect DLC relationship IDs for all base-game detail responses.""" - by_app: dict[int, list[tuple[int, float]]] = {} - unique_dlc_ids: set[int] = set() - - for result, game_data in zip(valid, details, strict=False): - if game_data is None: - continue - dlc_rels = _extract_dlc_relationships(game_data) - by_app[result.app_id] = dlc_rels - for dlc_id, _fallback_hours in dlc_rels: - if dlc_id > 0: - unique_dlc_ids.add(dlc_id) - - return by_app, sorted(unique_dlc_ids) + if not done % _SAVE_INTERVAL: + save_hltb_cache(cache, polls, extras) async def _fetch_dlc_leisure_hours( diff --git a/steam_backlog_enforcer/_hltb_matching.py b/steam_backlog_enforcer/_hltb_matching.py index 2bba01b..d0addd3 100644 --- a/steam_backlog_enforcer/_hltb_matching.py +++ b/steam_backlog_enforcer/_hltb_matching.py @@ -175,6 +175,7 @@ def _find_best_extended( share the base title as compilations (e.g. "FAITH: The Unholy Trinity"). """ best: tuple[dict[str, Any], float] | None = None + best_comp: Any = None # comp_100 of *best*; None until one is chosen for entry, sim in usable: game_type = str(entry.get("game_type", "")).lower() if game_type not in ("", "game", "compil"): @@ -184,11 +185,10 @@ def _find_best_extended( suffix = entry_name[len(lower) :].lstrip(" :-") if any(suffix.startswith(kw) for kw in _SUBSET_SUFFIXES): continue - if best is not None and entry.get("comp_100", 0) <= best[0].get( - "comp_100", 0 - ): + comp = entry.get("comp_100", 0) + if best_comp is not None and comp <= best_comp: continue - best = (entry, sim) + best, best_comp = (entry, sim), comp return best diff --git a/steam_backlog_enforcer/_hltb_search.py b/steam_backlog_enforcer/_hltb_search.py index e91deb6..f960071 100644 --- a/steam_backlog_enforcer/_hltb_search.py +++ b/steam_backlog_enforcer/_hltb_search.py @@ -14,6 +14,7 @@ from steam_backlog_enforcer._hltb_detail import ( ) from steam_backlog_enforcer._hltb_failures import ( SKIPPED_AFTER_RATE_LIMIT, + Breaker, HLTBUnavailableError, record_breaker, record_failure, @@ -31,6 +32,7 @@ from steam_backlog_enforcer._hltb_types import ( MAX_CONCURRENT, HLTBResult, ProgressCb, + _AuthInfo, _HLTBExtras, ) @@ -75,6 +77,42 @@ def _check_batch(ctx: _SearchCtx) -> None: raise _fail_batch(msg) +async def _acquire_auth( + search_url: str, client_timeout: aiohttp.ClientTimeout, breaker: Breaker +) -> _AuthInfo: + """Fetch the HLTB auth token through its own short-lived session. + + Raises: + HLTBUnavailableError: no token; a 429 also trips ``breaker``. + """ + auth_failures: list[str] = [] + async with aiohttp.ClientSession(timeout=client_timeout) as init_session: + auth = await _get_auth_info(search_url, init_session, auth_failures) + if auth is None: + forget_hltb_search_url() + why = auth_failures[0] if auth_failures else "no token" + if why == status_reason(HTTPStatus.TOO_MANY_REQUESTS): + breaker.tripped = True + msg = f"HLTB: rate-limited, nothing refreshed (auth init: {why})" + raise _fail_batch(msg) + logger.info("HLTB auth token acquired.") + return auth + + +def _search_headers(auth: _AuthInfo) -> dict[str, str]: + """Return the headers every search request in a batch shares.""" + headers: dict[str, str] = { + "content-type": "application/json", + "accept": "*/*", + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0" + ), + "referer": "https://howlongtobeat.com/", + } + apply_auth(headers, auth) + return headers + + async def _search_batch( games: list[tuple[int, str]], cache: dict[int, float], @@ -103,43 +141,19 @@ async def _search_batch( timeout = aiohttp.ClientTimeout(total=20, sock_read=15) # 2. Get auth info (separate session — avoids reuse issues). - auth_failures: list[str] = [] - async with aiohttp.ClientSession(timeout=timeout) as init_session: - auth = await _get_auth_info(search_url, init_session, auth_failures) - if auth is None: - forget_hltb_search_url() - why = auth_failures[0] if auth_failures else "no token" - if why == status_reason(HTTPStatus.TOO_MANY_REQUESTS): - breaker.tripped = True - msg = f"HLTB: rate-limited, nothing refreshed (auth init: {why})" - raise _fail_batch(msg) - logger.info("HLTB auth token acquired.") + auth = await _acquire_auth(search_url, timeout, breaker) - # 3. Build shared headers for all search requests. - headers: dict[str, str] = { - "content-type": "application/json", - "accept": "*/*", - "User-Agent": ( - "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0" - ), - "referer": "https://howlongtobeat.com/", - } - apply_auth(headers, auth) - - # 4. Fire all searches through a single persistent session. + # 3. Fire all searches through a single persistent session, sharing the + # auth-bearing headers. sem = asyncio.Semaphore(MAX_CONCURRENT) - connector = aiohttp.TCPConnector( - limit=MAX_CONCURRENT, - keepalive_timeout=30, - ) async with aiohttp.ClientSession( timeout=timeout, - connector=connector, + connector=aiohttp.TCPConnector(limit=MAX_CONCURRENT, keepalive_timeout=30), ) as session: ctx = _SearchCtx( session=session, search_url=search_url, - headers=headers, + headers=_search_headers(auth), cache=cache, polls=polls, extras=extras, @@ -168,7 +182,7 @@ async def _fetch_batch( extras = _HLTBExtras() search_results = await _search_batch(games, cache, polls, progress_cb, extras) - # 5. Fetch leisure times + DLC from game detail pages. + # 4. Fetch leisure times + DLC from game detail pages. logger.info( "Fetching leisure times for %d games from detail pages...", len(search_results), diff --git a/steam_backlog_enforcer/_hltb_types.py b/steam_backlog_enforcer/_hltb_types.py index bcd5588..39e3d06 100644 --- a/steam_backlog_enforcer/_hltb_types.py +++ b/steam_backlog_enforcer/_hltb_types.py @@ -144,7 +144,7 @@ def restore_prior_hours(refreshed: dict[int, float], prior: dict[int, float]) -> we already had -- prior leisure/DLC figures are trusted over a miss. """ for app_id, old_hours in prior.items(): - if old_hours > 0 and refreshed.get(app_id, -1.0) <= 0: + if refreshed.get(app_id, -1.0) <= 0 < old_hours: refreshed[app_id] = old_hours diff --git a/steam_backlog_enforcer/_leisure_kill.py b/steam_backlog_enforcer/_leisure_kill.py index d15cdf0..a48898f 100644 --- a/steam_backlog_enforcer/_leisure_kill.py +++ b/steam_backlog_enforcer/_leisure_kill.py @@ -130,8 +130,6 @@ class LeisureKiller: ) -> None: """Signal this tick's leisure targets if the leisure budget is spent. - Never raises: the enforce loop has no per-tick guard. - Args: state: Accounting state after this tick's policy. rules: Policy for this tick (``leisure`` set by the billing step). @@ -140,7 +138,9 @@ class LeisureKiller: """ try: self._enforce(state, rules, verdict=verdict, stamp=stamp) - except Exception as exc: # the loop must outlive any kill bug + # Never raises: the enforce loop has no per-tick guard and must outlive + # any kill bug, so this catch-all is on purpose. + except Exception as exc: # pylint: disable=broad-exception-caught message = f"{type(exc).__name__}: {exc}" if message != self._last_error: self._last_error = message @@ -179,6 +179,12 @@ class LeisureKiller: "day_key": state.day_key, "seconds": round(state.seconds, 3), } + self._escalate(live, base, stamp) + + def _escalate( + self, live: dict[_Ident, str], base: dict[str, object], stamp: float + ) -> None: + """SIGTERM each live target once, then SIGKILL it after the grace.""" for ident, key in sorted(live.items()): note = {"key": key, "name": exe_name(ident[0]), **base} first = self._termed.get(ident) diff --git a/steam_backlog_enforcer/_shadow.py b/steam_backlog_enforcer/_shadow.py index 796885d..8a19945 100644 --- a/steam_backlog_enforcer/_shadow.py +++ b/steam_backlog_enforcer/_shadow.py @@ -120,7 +120,8 @@ def observe_shadow(owners: dict[int, str], *, stamp: float) -> ShadowVerdict: if paused is not None: return ShadowVerdict(paused_by=paused, keys=()) fact = read_now(NOW_PATH, stamp, max_age=_MAX_AGE_SECONDS) - except Exception as exc: # the loop must outlive any shadow input bug + # Catch-all on purpose: the loop must outlive any shadow input bug. + except Exception as exc: # pylint: disable=broad-exception-caught if _should_warn(exc): logger.warning( "Shadow inputs unreadable (%r); failing closed.", exc, exc_info=True diff --git a/steam_backlog_enforcer/_stats.py b/steam_backlog_enforcer/_stats.py index 2d824c1..8a9aff0 100644 --- a/steam_backlog_enforcer/_stats.py +++ b/steam_backlog_enforcer/_stats.py @@ -125,12 +125,46 @@ def _print_player_speed_scenario( _echo(f" @ {daily:.0f} h/day → {estimate}") +def _print_header( + qualified: list[_GameTimes], skips: list[int], state: State, games_done: int +) -> None: + """Print the banner, the filter counts and the excluded games.""" + hltb_skip, linux_skip, no_data_skip = skips + total_q = len(qualified) + rule = "═" * 70 + _echo(f"\n{rule}\n BACKLOG COMPLETION ESTIMATES\n{rule}") + _echo(f"\n Qualifying games: {total_q}") + if hltb_skip: + _echo(f" HLTB-skipped: {hltb_skip} (confidence too low)") + if linux_skip: + _echo(f" Linux-skipped: {linux_skip} (poor ProtonDB rating)") + if no_data_skip: + _echo(f" No-data-skipped: {no_data_skip} (no HLTB hours at all)") + + missing_rush_final = sum(1 for e in qualified if e.rush_hours <= 0) + if missing_rush_final: + _echo( + f"\n Note: {missing_rush_final}/{total_q} games still missing" + " rush/leisure data (HLTB search may not have matched them)." + ) + elif total_q: + _echo( + f"\n Detail data: rush + leisure available for all {total_q}" + " qualifying games." + ) + + if state.current_app_id: + _echo( + f"\n Current game: {state.current_game_name} (excluded from totals)" + ) + _echo(f" Finished games: {games_done} (excluded from totals)") + + def cmd_stats(_config: Config, state: State) -> None: """Display backlog completion-time statistics. - Filters games by the same HLTB-confidence and Linux-compatibility rules - used when picking the next game. Auto-fetches missing rush/leisure detail - data before printing. Shows five scenarios: + Filters games by the HLTB-confidence and Linux-compatibility rules used to + pick the next game, auto-fetching missing rush/leisure data. Shows: 1. At your current pace (games finished per day since enforcement started). 2. Rush — avg comp_100 + DLC completion time per HLTB. @@ -138,8 +172,8 @@ def cmd_stats(_config: Config, state: State) -> None: 4. Worst — absolute maximum recorded time (any category) per HLTB. 5. Your play style — extrapolated from completed-game calibration vs HLTB. - Ends with one line per kind of HLTB failure (the cache is kept for those - games) and exits 1 when some HLTB batch refreshed nothing at all. + Ends with one line per HLTB failure kind (their cache is kept) and exits 1 + when some HLTB batch refreshed nothing at all. """ reset_failures() try: @@ -170,44 +204,12 @@ def _print_stats(_config: Config, state: State) -> None: # Ensure completed games have rush/leisure data for pace calibration. _ensure_completed_rush_data(games) - qualified, hltb_skip, linux_skip, no_data_skip = _filter_qualifying_games( - games, state - ) + qualified, *skips = _filter_qualifying_games(games, state) if _ensure_rush_data(qualified): # Re-filter picks up updated rush/leisure caches; ProtonDB is now cached. - qualified, hltb_skip, linux_skip, no_data_skip = _filter_qualifying_games( - games, state - ) + qualified, *skips = _filter_qualifying_games(games, state) total_q = len(qualified) - - _echo(f"\n{'═' * 70}") - _echo(" BACKLOG COMPLETION ESTIMATES") - _echo(f"{'═' * 70}") - _echo(f"\n Qualifying games: {total_q}") - if hltb_skip: - _echo(f" HLTB-skipped: {hltb_skip} (confidence too low)") - if linux_skip: - _echo(f" Linux-skipped: {linux_skip} (poor ProtonDB rating)") - if no_data_skip: - _echo(f" No-data-skipped: {no_data_skip} (no HLTB hours at all)") - - missing_rush_final = sum(1 for e in qualified if e.rush_hours <= 0) - if missing_rush_final: - _echo( - f"\n Note: {missing_rush_final}/{total_q} games still missing" - " rush/leisure data (HLTB search may not have matched them)." - ) - elif total_q: - _echo( - f"\n Detail data: rush + leisure available for all {total_q}" - " qualifying games." - ) - - if state.current_app_id: - _echo( - f"\n Current game: {state.current_game_name} (excluded from totals)" - ) - _echo(f" Finished games: {games_done} (excluded from totals)") + _print_header(qualified, skips, state, games_done) _echo(f"\n{_LINE}") _print_pace_scenario(state, total_q, games_done_since_start) @@ -242,8 +244,7 @@ def _print_stats(_config: Config, state: State) -> None: _print_worst_example(qualified) # Pace calibration uses the freshly-updated cache (both fetches above ran). - raw_cache = _read_raw_cache() - pace_vs_hltb = compute_pace_vs_hltb(games, raw_cache) + pace_vs_hltb = compute_pace_vs_hltb(games, _read_raw_cache()) _print_player_speed_scenario(pace_vs_hltb, rush_total, leisure_total) _echo(f"\n{_LINE}\n") diff --git a/steam_backlog_enforcer/_stats_gathering.py b/steam_backlog_enforcer/_stats_gathering.py index f03f849..d5a5560 100644 --- a/steam_backlog_enforcer/_stats_gathering.py +++ b/steam_backlog_enforcer/_stats_gathering.py @@ -7,6 +7,7 @@ the 250-line cap. Leaf helpers: nothing here calls back into ``_stats``. from __future__ import annotations from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass import logging from typing import TYPE_CHECKING @@ -41,6 +42,46 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class _HltbCaches: + """The HLTB caches the stats filter reads, loaded once per filter run.""" + + rush: dict[int, float] + leisure_100h: dict[int, float] + game_id: dict[int, int] + hours: dict[int, float] + + @classmethod + def load(cls) -> _HltbCaches: + """Read every cache from disk.""" + return cls( + rush=load_hltb_rush_cache(), + leisure_100h=load_hltb_leisure_100h_cache(), + game_id=load_hltb_game_id_cache(), + hours=load_hltb_cache(), + ) + + def times_for(self, game: GameInfo) -> _GameTimes | None: + """*game*'s time estimates, or None when HLTB has no time for it.""" + rush = self.rush.get(game.app_id, -1) + leisure = self.leisure_100h.get(game.app_id, -1) + # worst_hours = max of: snapshot completionist, HLTB hours cache (fallback + # when snapshot is stale/missing), and leisure_100h (slowest 100% time). + snap_hours = game.completionist_hours if game.completionist_hours > 0 else -1 + cache_hours = self.hours.get(game.app_id, -1) + worst_candidates = [v for v in (snap_hours, cache_hours, leisure) if v > 0] + worst = max(worst_candidates) if worst_candidates else -1 + if worst <= 0 and rush <= 0 and leisure <= 0: + return None + return _GameTimes( + game=game, + worst_hours=worst, + rush_hours=rush, + leisure_100h=leisure, + hltb_game_id=self.game_id.get(game.app_id, 0), + ) + + def _filter_qualifying_games( games: list[GameInfo], state: State, @@ -53,10 +94,7 @@ def _filter_qualifying_games( Returns: (qualified_list, hltb_skipped, linux_skipped, no_data_skipped) """ - rush_cache = load_hltb_rush_cache() - leisure_100h_cache = load_hltb_leisure_100h_cache() - game_id_cache = load_hltb_game_id_cache() - hours_cache = load_hltb_cache() + caches = _HltbCaches.load() exclude = set(state.finished_app_ids) if state.current_app_id is not None: @@ -66,53 +104,25 @@ def _filter_qualifying_games( _apply_cached_confidence_to_candidates(candidates) _refresh_candidate_confidence_batch(candidates) - hltb_skipped = 0 - linux_skipped = 0 - no_data_skipped = 0 - app_ids_to_check: list[int] = [] - - conf_ok: list[GameInfo] = [] - for game in candidates: - if _confidence_fail_reasons(game): - hltb_skipped += 1 - continue - conf_ok.append(game) - app_ids_to_check.append(game.app_id) + conf_ok = [game for game in candidates if not _confidence_fail_reasons(game)] + hltb_skipped = len(candidates) - len(conf_ok) ratings: dict[int, ProtonDBRating] = {} - if app_ids_to_check: - ratings = fetch_protondb_ratings(app_ids_to_check) + if conf_ok: + ratings = fetch_protondb_ratings([game.app_id for game in conf_ok]) + linux_skipped = 0 + no_data_skipped = 0 qualified: list[_GameTimes] = [] for game in conf_ok: - rating = ratings.get(game.app_id, ProtonDBRating(app_id=game.app_id)) - if not rating.is_playable: + if not ratings.get(game.app_id, ProtonDBRating(app_id=game.app_id)).is_playable: linux_skipped += 1 continue - - rush = rush_cache.get(game.app_id, -1) - leisure = leisure_100h_cache.get(game.app_id, -1) - - # worst_hours = max of: snapshot completionist, HLTB hours cache (fallback - # when snapshot is stale/missing), and leisure_100h (slowest 100% time). - snap_hours = game.completionist_hours if game.completionist_hours > 0 else -1 - cache_hours = hours_cache.get(game.app_id, -1) - worst_candidates = [v for v in (snap_hours, cache_hours, leisure) if v > 0] - worst = max(worst_candidates) if worst_candidates else -1 - - if worst <= 0 and rush <= 0 and leisure <= 0: + times = caches.times_for(game) + if times is None: no_data_skipped += 1 continue - - qualified.append( - _GameTimes( - game=game, - worst_hours=worst, - rush_hours=rush, - leisure_100h=leisure, - hltb_game_id=game_id_cache.get(game.app_id, 0), - ) - ) + qualified.append(times) return qualified, hltb_skipped, linux_skipped, no_data_skipped diff --git a/steam_backlog_enforcer/_steam_models.py b/steam_backlog_enforcer/_steam_models.py index fcf14c6..90f1d98 100644 --- a/steam_backlog_enforcer/_steam_models.py +++ b/steam_backlog_enforcer/_steam_models.py @@ -46,10 +46,7 @@ class GameInfo: @property def is_complete(self) -> bool: """True if all achievements are unlocked.""" - return ( - self.total_achievements > 0 - and self.unlocked_achievements >= self.total_achievements - ) + return 0 < self.total_achievements <= self.unlocked_achievements def to_snapshot(self) -> dict[str, Any]: """Serialize to JSON-safe dict.""" diff --git a/steam_backlog_enforcer/_web_routes.py b/steam_backlog_enforcer/_web_routes.py index db12911..e088597 100644 --- a/steam_backlog_enforcer/_web_routes.py +++ b/steam_backlog_enforcer/_web_routes.py @@ -84,7 +84,7 @@ def resolve(method: str, path: str) -> tuple[Route, tuple[str, ...]] | str | Non """ path_known = False for entry in ROUTES: - match = entry.pattern.fullmatch(path) + match = re.fullmatch(entry.pattern, path) if match is None: continue if entry.method == method: diff --git a/steam_backlog_enforcer/_web_server.py b/steam_backlog_enforcer/_web_server.py index 400a2a7..cf5b19d 100644 --- a/steam_backlog_enforcer/_web_server.py +++ b/steam_backlog_enforcer/_web_server.py @@ -101,7 +101,8 @@ class _Handler(BaseHTTPRequestHandler): self._dispatch(method, split.path, _query(split.query)) except ApiError as exc: send_error(self, exc) - except Exception: + # Catch-all on purpose: any bug answers 500, the server keeps serving. + except Exception: # pylint: disable=broad-exception-caught logger.exception("%s %s failed", method, split.path) msg = "The server failed to answer this request; see its log." send_error( diff --git a/steam_backlog_enforcer/_web_sse.py b/steam_backlog_enforcer/_web_sse.py index 1d5477b..fffcce1 100644 --- a/steam_backlog_enforcer/_web_sse.py +++ b/steam_backlog_enforcer/_web_sse.py @@ -14,6 +14,7 @@ The server only reads files here, so a job is unaffected by any of this. from __future__ import annotations +import contextlib from http import HTTPStatus import json import time @@ -106,9 +107,8 @@ def stream_job_events(handler: BaseHTTPRequestHandler, request: Request) -> None handler.send_header("Cache-Control", "no-store") handler.send_header("X-Accel-Buffering", "no") handler.end_headers() - try: + # The browser going away is not an error; it will reconnect if it cares. + with contextlib.suppress(BrokenPipeError, ConnectionResetError): handler.wfile.write(f"retry: {_RETRY_MS}\n\n".encode()) _pump(handler, job_id, after) handler.wfile.flush() - except BrokenPipeError, ConnectionResetError: - return # The browser went away; it will reconnect if it cares. diff --git a/steam_backlog_enforcer/jobs/_runner.py b/steam_backlog_enforcer/jobs/_runner.py index 337cc7e..b8fb6a5 100644 --- a/steam_backlog_enforcer/jobs/_runner.py +++ b/steam_backlog_enforcer/jobs/_runner.py @@ -163,8 +163,11 @@ def _run_command( # The gate already printed its banner (now log events); the # summary is the one-line form of the same refusal. sink.flush() - reason = lock_reason(command, config, State.load()) - return _failed(reason or "Refused by the lock gate.", EXIT_REFUSED) + return _failed( + lock_reason(command, config, State.load()) + or "Refused by the lock gate.", + EXIT_REFUSED, + ) try: data = job(params, progress, prompter) except SystemExit as exc: @@ -205,10 +208,12 @@ def run_job(job_dir: Path) -> int: outcome = _failed(exc.message, EXIT_REFUSED) except PrivilegedViaDaemonError as exc: outcome = _failed(str(exc), EXIT_REFUSED) - except Exception as exc: + # Catch-all on purpose: every job ends with a result, however it fails. + except Exception as exc: # pylint: disable=broad-exception-caught logger.exception("Job %s crashed", job_dir.name) outcome = _failed(f"{type(exc).__name__}: {exc}") - except BaseException as exc: # SystemExit from outside a handler, etc. + # SystemExit from outside a handler, etc.: still write the result. + except BaseException as exc: # pylint: disable=broad-exception-caught logger.exception("Job %s stopped", job_dir.name) outcome = _failed(f"{type(exc).__name__}: {exc}") _finish(job_dir, writer, outcome) diff --git a/steam_backlog_enforcer/jobs/_spawn.py b/steam_backlog_enforcer/jobs/_spawn.py index 41048fd..aedca06 100644 --- a/steam_backlog_enforcer/jobs/_spawn.py +++ b/steam_backlog_enforcer/jobs/_spawn.py @@ -18,7 +18,7 @@ import shutil import subprocess import sys import time -from typing import Final +from typing import BinaryIO, Final from steam_backlog_enforcer.config import _atomic_write from steam_backlog_enforcer.jobs._errors import JobError @@ -111,6 +111,25 @@ def _confirm_started(proc: subprocess.Popen[bytes], path: Path) -> None: time.sleep(_POLL_SECONDS) +def _start_session( + path: Path, out: BinaryIO, env: dict[str, str], lock_fd: int | None +) -> subprocess.Popen[bytes]: + """Start the job process in its own session; the caller owns it. + + No ``with``: the job outlives the CLI call that spawned it. + """ + return subprocess.Popen( + _spawn_argv(path), + stdin=subprocess.DEVNULL, + stdout=out, + stderr=out, + cwd=_PACKAGE_ROOT, + env=env, + start_new_session=True, + pass_fds=() if lock_fd is None else (lock_fd,), + ) + + def spawn_job(path: Path, lock_fd: int | None) -> int: """Start the detached job process for *path* and return its pid. @@ -130,16 +149,7 @@ def spawn_job(path: Path, lock_fd: int | None) -> int: env[LOCK_FD_ENV] = str(lock_fd) try: with (path / OUTPUT_NAME).open("ab") as out: - proc = subprocess.Popen( - _spawn_argv(path), - stdin=subprocess.DEVNULL, - stdout=out, - stderr=out, - cwd=_PACKAGE_ROOT, - env=env, - start_new_session=True, - pass_fds=() if lock_fd is None else (lock_fd,), - ) + proc = _start_session(path, out, env, lock_fd) except OSError as exc: msg = f"The job process could not be started: {exc}." raise _fail_unstarted(path, msg) from exc diff --git a/steam_backlog_enforcer/library_hider.py b/steam_backlog_enforcer/library_hider.py index fa9a2ef..5d1955e 100644 --- a/steam_backlog_enforcer/library_hider.py +++ b/steam_backlog_enforcer/library_hider.py @@ -21,7 +21,7 @@ import logging from typing import TYPE_CHECKING from steam_backlog_enforcer._achievement_skip import zero_achievement_ids -from steam_backlog_enforcer._actions import allowed_app_ids +from steam_backlog_enforcer._actions import allowed_app_ids as _allowed_ids_of from steam_backlog_enforcer._cdp import _cdp_result_value, _evaluate_js from steam_backlog_enforcer._echo import _echo from steam_backlog_enforcer._owned_apps_cache import get_all_owned_app_ids @@ -233,7 +233,7 @@ def hide_others_and_report(config: Config, state: State, *, lead: str = "\n ") owned_ids = get_all_owned_app_ids(config) if not owned_ids: return - hidden, skipped = try_hide_other_games(owned_ids, allowed_app_ids(state)) + hidden, skipped = try_hide_other_games(owned_ids, _allowed_ids_of(state)) if skipped is not None: _echo(f"{lead}Library hiding: skipped ({skipped})") elif hidden > 0: diff --git a/steam_backlog_enforcer/scanning.py b/steam_backlog_enforcer/scanning.py index 1ac9264..8ee8bae 100644 --- a/steam_backlog_enforcer/scanning.py +++ b/steam_backlog_enforcer/scanning.py @@ -78,11 +78,8 @@ def do_scan(config: Config, state: State) -> list[GameInfo]: client = SteamAPIClient(config.steam_api_key, config.steam_id) start = time.time() - done_count = 0 def progress(current: int, total: int) -> None: - nonlocal done_count - done_count = current if current % 50 == 0 or current == total: _echo(f"\r Scanning achievements: {current}/{total}", end="", flush=True) @@ -115,19 +112,19 @@ def do_scan(config: Config, state: State) -> list[GameInfo]: polls_cache = load_hltb_polls_cache() count_comp_cache = load_hltb_count_comp_cache() for g in games: - hours = hltb_cache.get(g.app_id, -1) - g.completionist_hours = hours + g.completionist_hours = hltb_cache.get(g.app_id, -1) g.comp_100_count = polls_cache.get(g.app_id, 0) g.count_comp = count_comp_cache.get(g.app_id, 0) - found = sum(1 for h in hltb_cache.values() if h > 0) - _echo(f" HLTB data: {found} games have completion estimates") + _echo( + f" HLTB data: {sum(1 for h in hltb_cache.values() if h > 0)} games" + " have completion estimates" + ) # Save snapshot. save_snapshot([g.to_snapshot() for g in games]) - complete = [g for g in games if g.is_complete] - incomplete_games = [g for g in games if not g.is_complete] - _echo(f"\nResults: {len(complete)} complete, {len(incomplete_games)} incomplete") + n_complete = sum(1 for g in games if g.is_complete) + _echo(f"\nResults: {n_complete} complete, {len(games) - n_complete} incomplete") # Auto-pick a game if none assigned. if state.current_app_id is None: diff --git a/steam_backlog_enforcer/store_blocker.py b/steam_backlog_enforcer/store_blocker.py index 9a78257..d6865c8 100644 --- a/steam_backlog_enforcer/store_blocker.py +++ b/steam_backlog_enforcer/store_blocker.py @@ -37,6 +37,9 @@ from steam_backlog_enforcer.config import ( logger = logging.getLogger(__name__) +# The install script rewrites /etc/hosts and re-applies its guards. +_HOSTS_INSTALL_TIMEOUT_SECONDS = 120 + def is_store_blocked() -> bool: """Check if Steam Store domains are blocked in /etc/hosts or iptables.""" @@ -110,7 +113,7 @@ def _block_via_hosts_install() -> bool: [SUDO, BASH, str(HOSTS_INSTALL_SCRIPT), "--no-flush-dns"], capture_output=True, text=True, - timeout=120, + timeout=_HOSTS_INSTALL_TIMEOUT_SECONDS, check=False, ) except OSError, subprocess.SubprocessError: