Pylint gate 10 over the whole package
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (mypy, mypy, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (pylint, pylint, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (rest, , mypy,pylint) (push) Waiting to run
Tests / test (push) Waiting to run
Tests / web-coverage (push) Waiting to run
repo-contract / repo contract (push) Waiting to run

Raise the pylint hook to --fail-under=10.0 and make it lint the whole
package on every run (pass_filenames: false): over the staged subset a
9.99 tree passed at commit time.

Ports de2aea3 (9b9342ba-1) onto main, minus its web/ and requirements.txt
bumps and the iptables dedupe f1822f1 already landed; _hltb_confidence and
_hltb_search keep main's 429 handling. _search_batch splits out
_acquire_auth / _search_headers, and _stats.py is back to 250 lines.

Drops the min-similarity-lines = 8 floor (0 duplicate-code hits at the
default) and sets max-attributes = 15 for the flat persisted records, as
kuhy decided. Tests stay outside pylint, as at every site already at 10.

Todo: fda71ed1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Krzysztof Rudnicki 2026-11-10 16:57:35 +01:00
parent f1822f154f
commit 57d262b711
25 changed files with 336 additions and 259 deletions

View File

@ -124,13 +124,19 @@ repos:
rev: v4.0.9
hooks:
- id: pylint
exclude: ^scripts/file_length/
# Whole tree on every run, never the staged subset: `--fail-under`
# grades a score, and a score over whatever happened to be staged let a
# 9.99 tree pass at commit time and fail in CI. scripts/file_length/
# is the shared copy from ~/src/utils, linted there.
pass_filenames: false
args:
- --rcfile=pyproject.toml
- --fail-under=8.0
- --fail-under=10.0
# Serial: parallel pylint pickles PyLinter with dill, which crashes on
# Python 3.15 (removed co_lnotab). Same reason as jobs = 1 in pyproject.
- --jobs=1
- steam_backlog_enforcer
- scripts/check_requirements_sync.py
# pylint resolves imports in its OWN isolated env, so anything the
# package imports has to be listed here or it reports import-error and
# drags the score under --fail-under. `mcp` is a real runtime import

View File

@ -169,15 +169,14 @@ disable = [
"use-implicit-booleaness-not-comparison-to-zero",
]
[tool.pylint.similarities]
# Same duplicate-code threshold as diet-guard: shorter matches are mostly
# import blocks and subprocess.run keyword lists, not real copy-paste.
min-similarity-lines = 8
[tool.pylint.design]
min-public-methods = 0
max-module-lines = 250
max-attributes = 10
# 15 = the widest persisted record (config.Config). Config, State and
# PlaytimeState are flat JSON files the live daemon reads and writes; WebGame
# is the UI's row shape. Splitting them under 10 would change the on-disk shape
# or hide it behind property forwarders written only for this check.
max-attributes = 15
[tool.pylint.typecheck]
generated-members = [

View File

@ -151,7 +151,7 @@ class CtlServer:
logger.warning("Control socket: refused a connection from uid %d", uid)
conn.close()
return
if not self._slots.acquire(blocking=False):
if not self._take_slot():
logger.warning("Control socket: too many connections, dropping one")
conn.close()
return
@ -159,13 +159,21 @@ class CtlServer:
target=self._handle, args=(conn, uid), name="ctl-conn", daemon=True
).start()
def _take_slot(self) -> bool:
"""Claim a connection slot without blocking; ``_handle`` releases it.
No ``with``: the slot is released by the worker thread, not here.
"""
return self._slots.acquire(blocking=False)
def _handle(self, conn: socket.socket, uid: int) -> None:
"""Serve one connection, always releasing its slot and socket."""
try:
line = read_line(conn)
if line is not None:
self._write(conn, self._dispatch(line, uid))
except Exception:
# Catch-all on purpose: a dying connection must not kill its thread.
except Exception: # pylint: disable=broad-exception-caught
logger.debug("Control connection dropped", exc_info=True)
finally:
conn.close()
@ -184,7 +192,8 @@ class CtlServer:
except CtlError as exc:
logger.info("Control socket: refused (%s)", exc.code)
return error_response(exc)
except Exception:
# Catch-all on purpose: any op bug answers "internal error", never raises.
except Exception: # pylint: disable=broad-exception-caught
logger.exception("Control socket op crashed")
return error_response(CtlError(OP_FAILED, "internal error"))

View File

@ -46,6 +46,7 @@ from steam_backlog_enforcer._steam_state import is_game_fully_installed
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from typing import BinaryIO
logger = logging.getLogger(__name__)
@ -134,6 +135,21 @@ def poll_fast_installs() -> None:
_fip.poll_trackers({run.app_id for run in _RUNS}, is_game_fully_installed)
def _start_session(cmd: list[str], log: BinaryIO) -> subprocess.Popen[bytes]:
"""Start *cmd* in its own session writing to *log*; the caller owns it.
No ``with``: the run outlives this call and is reaped by
:func:`poll_fast_installs`.
"""
return subprocess.Popen(
cmd,
stdin=subprocess.DEVNULL,
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
def _spawn_detached(cmd: list[str], app_id: int, label: str = "") -> None:
"""Start the installer in its own session, output to a log file."""
# Per user: the daemon's log is root-owned, and a web job runs as the
@ -149,13 +165,7 @@ def _spawn_detached(cmd: list[str], app_id: int, label: str = "") -> None:
with log_path.open("ab") as log:
# Append mode starts at the end: earlier runs' lines are never parsed.
_fip.track(app_id, label, log_path, offset=log.tell(), now=time.monotonic())
proc = subprocess.Popen(
cmd,
stdin=subprocess.DEVNULL,
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
proc = _start_session(cmd, log)
_RUNS.append(_DetachedRun(app_id, proc, log_path))
logger.info("Started steam-game-installer for AppID=%d (log: %s)", app_id, log_path)

View File

@ -31,6 +31,7 @@ the daemon's own enforce loop polls the trackers.
from __future__ import annotations
import contextlib
from dataclasses import dataclass
import re
import threading
@ -109,16 +110,13 @@ class InstallTracker:
A log that is missing or unreadable changes nothing: the run itself is
judged by its exit status, never by its log.
"""
try:
with self._path.open("rb") as log:
log.seek(self._offset)
while chunk := log.read(_READ_CHUNK):
self._offset += len(chunk)
*lines, self._partial = (self._partial + chunk).split(b"\n")
for line in lines:
self._feed(line.decode("utf-8", "replace").rstrip("\r"))
except OSError:
return
with contextlib.suppress(OSError), self._path.open("rb") as log:
log.seek(self._offset)
while chunk := log.read(_READ_CHUNK):
self._offset += len(chunk)
*lines, self._partial = (self._partial + chunk).split(b"\n")
for line in lines:
self._feed(line.decode("utf-8", "replace").rstrip("\r"))
def _feed(self, line: str) -> None:
"""Apply one log line to the phase / percent state."""

View File

@ -69,6 +69,7 @@ def _protected_name_stems() -> list[str]:
"""
try:
return [name for _, name in allowed_games(State.load()) if name]
# Catch-all on purpose: a best-effort safety net, not a hard dependency.
except Exception: # pylint: disable=broad-exception-caught
logger.exception("Could not load allowed games for the deletion safety net")
return []

View File

@ -31,6 +31,8 @@ from steam_backlog_enforcer._hltb_types import (
if TYPE_CHECKING:
import asyncio
from steam_backlog_enforcer._hltb_types import HLTBResult
logger = logging.getLogger(__name__)
_NEXT_DATA_RE = re.compile(
@ -130,3 +132,23 @@ async def _fetch_detail_one(
)
record_failure(f"detail {reason}")
return None
def _collect_dlc_relationships(
valid: list[HLTBResult],
details: list[dict[str, Any] | None],
) -> tuple[dict[int, list[tuple[int, float]]], list[int]]:
"""Collect DLC relationship IDs for all base-game detail responses."""
by_app: dict[int, list[tuple[int, float]]] = {}
unique_dlc_ids: set[int] = set()
for result, game_data in zip(valid, details, strict=False):
if game_data is None:
continue
dlc_rels = _extract_dlc_relationships(game_data)
by_app[result.app_id] = dlc_rels
for dlc_id, _fallback_hours in dlc_rels:
if dlc_id > 0:
unique_dlc_ids.add(dlc_id)
return by_app, sorted(unique_dlc_ids)

View File

@ -120,6 +120,23 @@ def fetch_hltb_confidence_cached(
return cache
def _log_detail_plan(n_rush: int, n_id: int) -> None:
"""Log what a detail fetch is about to do."""
if n_rush and n_id:
logger.info(
"Fetching HLTB detail for %d games missing rush/leisure data"
" + %d games missing game ID...",
n_rush,
n_id,
)
elif n_rush:
logger.info(
"Fetching HLTB detail for %d games missing rush/leisure data...", n_rush
)
else:
logger.info("Backfilling HLTB game ID for %d game(s)...", n_id)
def fetch_hltb_detail_missing(
games: list[tuple[int, str]],
progress_cb: ProgressCb | None = None,
@ -167,21 +184,7 @@ def fetch_hltb_detail_missing(
prior_hours = {aid: cache[aid] for aid, _ in missing if aid in cache}
n_rush = len(missing_rush)
n_id = len(missing_id_only)
if n_rush and n_id:
logger.info(
"Fetching HLTB detail for %d games missing rush/leisure data"
" + %d games missing game ID...",
n_rush,
n_id,
)
elif n_rush:
logger.info(
"Fetching HLTB detail for %d games missing rush/leisure data...", n_rush
)
else:
logger.info("Backfilling HLTB game ID for %d game(s)...", n_id)
_log_detail_plan(len(missing_rush), len(missing_id_only))
report = reporting_progress_cb("Fetching HLTB details", len(missing), progress_cb)
try:
elapsed = fetch_hltb_times_timed(missing, cache, polls, report, extras)

View File

@ -9,6 +9,7 @@ from typing import Any
import aiohttp
from steam_backlog_enforcer._hltb_comp_extract import (
_collect_dlc_relationships,
_extract_comp_100_avg_and_high,
_extract_dlc_relationships,
_fetch_detail_one,
@ -141,6 +142,27 @@ def _apply_detail_to_extras(
return leisure
async def _fetch_all_details(
valid: list[HLTBResult],
) -> tuple[
list[dict[str, Any] | None], dict[int, list[tuple[int, float]]], dict[int, float]
]:
"""Fetch every detail page, then the leisure hours of every DLC they list."""
timeout = aiohttp.ClientTimeout(total=30, sock_read=20)
sem = asyncio.Semaphore(MAX_CONCURRENT)
connector = aiohttp.TCPConnector(limit=MAX_CONCURRENT, keepalive_timeout=30)
async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
coros = [_fetch_detail_one(sem, session, r.hltb_game_id) for r in valid]
names = [r.game_name for r in valid]
details = await gather_tracked("Fetching HLTB detail pages", coros, names)
dlc_relationships_by_app, dlc_ids = _collect_dlc_relationships(valid, details)
return (
details,
dlc_relationships_by_app,
await _fetch_dlc_leisure_hours(sem, session, dlc_ids),
)
async def _fetch_leisure_times(
search_results: list[HLTBResult],
cache: dict[int, float],
@ -163,65 +185,27 @@ async def _fetch_leisure_times(
if not valid:
return
timeout = aiohttp.ClientTimeout(total=30, sock_read=20)
sem = asyncio.Semaphore(MAX_CONCURRENT)
connector = aiohttp.TCPConnector(
limit=MAX_CONCURRENT,
keepalive_timeout=30,
)
total = len(valid)
done = 0
details, dlc_relationships_by_app, dlc_hours_by_id = await _fetch_all_details(valid)
found = 0
for done, (r, game_data) in enumerate(zip(valid, details, strict=False), 1):
if game_data is not None:
leisure = _apply_detail_to_extras(
r.app_id,
game_data,
dlc_relationships_by_app.get(r.app_id, []),
dlc_hours_by_id,
extras,
)
if leisure > 0:
r.completionist_hours = leisure
cache[r.app_id] = leisure
found += 1
async with aiohttp.ClientSession(
timeout=timeout,
connector=connector,
) as session:
coros = [_fetch_detail_one(sem, session, r.hltb_game_id) for r in valid]
names = [r.game_name for r in valid]
details = await gather_tracked("Fetching HLTB detail pages", coros, names)
if progress_cb is not None:
progress_cb(done, len(valid), found, r.game_name)
dlc_relationships_by_app, dlc_ids = _collect_dlc_relationships(valid, details)
dlc_hours_by_id = await _fetch_dlc_leisure_hours(sem, session, dlc_ids)
for r, game_data in zip(valid, details, strict=False):
done += 1
if game_data is not None:
dlc_rels = dlc_relationships_by_app.get(r.app_id, [])
leisure = _apply_detail_to_extras(
r.app_id, game_data, dlc_rels, dlc_hours_by_id, extras
)
if leisure > 0:
r.completionist_hours = leisure
cache[r.app_id] = leisure
found += 1
if progress_cb is not None:
progress_cb(done, total, found, r.game_name)
if not done % _SAVE_INTERVAL:
save_hltb_cache(cache, polls, extras)
def _collect_dlc_relationships(
valid: list[HLTBResult],
details: list[dict[str, Any] | None],
) -> tuple[dict[int, list[tuple[int, float]]], list[int]]:
"""Collect DLC relationship IDs for all base-game detail responses."""
by_app: dict[int, list[tuple[int, float]]] = {}
unique_dlc_ids: set[int] = set()
for result, game_data in zip(valid, details, strict=False):
if game_data is None:
continue
dlc_rels = _extract_dlc_relationships(game_data)
by_app[result.app_id] = dlc_rels
for dlc_id, _fallback_hours in dlc_rels:
if dlc_id > 0:
unique_dlc_ids.add(dlc_id)
return by_app, sorted(unique_dlc_ids)
if not done % _SAVE_INTERVAL:
save_hltb_cache(cache, polls, extras)
async def _fetch_dlc_leisure_hours(

View File

@ -175,6 +175,7 @@ def _find_best_extended(
share the base title as compilations (e.g. "FAITH: The Unholy Trinity").
"""
best: tuple[dict[str, Any], float] | None = None
best_comp: Any = None # comp_100 of *best*; None until one is chosen
for entry, sim in usable:
game_type = str(entry.get("game_type", "")).lower()
if game_type not in ("", "game", "compil"):
@ -184,11 +185,10 @@ def _find_best_extended(
suffix = entry_name[len(lower) :].lstrip(" :-")
if any(suffix.startswith(kw) for kw in _SUBSET_SUFFIXES):
continue
if best is not None and entry.get("comp_100", 0) <= best[0].get(
"comp_100", 0
):
comp = entry.get("comp_100", 0)
if best_comp is not None and comp <= best_comp:
continue
best = (entry, sim)
best, best_comp = (entry, sim), comp
return best

View File

@ -14,6 +14,7 @@ from steam_backlog_enforcer._hltb_detail import (
)
from steam_backlog_enforcer._hltb_failures import (
SKIPPED_AFTER_RATE_LIMIT,
Breaker,
HLTBUnavailableError,
record_breaker,
record_failure,
@ -31,6 +32,7 @@ from steam_backlog_enforcer._hltb_types import (
MAX_CONCURRENT,
HLTBResult,
ProgressCb,
_AuthInfo,
_HLTBExtras,
)
@ -75,6 +77,42 @@ def _check_batch(ctx: _SearchCtx) -> None:
raise _fail_batch(msg)
async def _acquire_auth(
search_url: str, client_timeout: aiohttp.ClientTimeout, breaker: Breaker
) -> _AuthInfo:
"""Fetch the HLTB auth token through its own short-lived session.
Raises:
HLTBUnavailableError: no token; a 429 also trips ``breaker``.
"""
auth_failures: list[str] = []
async with aiohttp.ClientSession(timeout=client_timeout) as init_session:
auth = await _get_auth_info(search_url, init_session, auth_failures)
if auth is None:
forget_hltb_search_url()
why = auth_failures[0] if auth_failures else "no token"
if why == status_reason(HTTPStatus.TOO_MANY_REQUESTS):
breaker.tripped = True
msg = f"HLTB: rate-limited, nothing refreshed (auth init: {why})"
raise _fail_batch(msg)
logger.info("HLTB auth token acquired.")
return auth
def _search_headers(auth: _AuthInfo) -> dict[str, str]:
"""Return the headers every search request in a batch shares."""
headers: dict[str, str] = {
"content-type": "application/json",
"accept": "*/*",
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0"
),
"referer": "https://howlongtobeat.com/",
}
apply_auth(headers, auth)
return headers
async def _search_batch(
games: list[tuple[int, str]],
cache: dict[int, float],
@ -103,43 +141,19 @@ async def _search_batch(
timeout = aiohttp.ClientTimeout(total=20, sock_read=15)
# 2. Get auth info (separate session — avoids reuse issues).
auth_failures: list[str] = []
async with aiohttp.ClientSession(timeout=timeout) as init_session:
auth = await _get_auth_info(search_url, init_session, auth_failures)
if auth is None:
forget_hltb_search_url()
why = auth_failures[0] if auth_failures else "no token"
if why == status_reason(HTTPStatus.TOO_MANY_REQUESTS):
breaker.tripped = True
msg = f"HLTB: rate-limited, nothing refreshed (auth init: {why})"
raise _fail_batch(msg)
logger.info("HLTB auth token acquired.")
auth = await _acquire_auth(search_url, timeout, breaker)
# 3. Build shared headers for all search requests.
headers: dict[str, str] = {
"content-type": "application/json",
"accept": "*/*",
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0"
),
"referer": "https://howlongtobeat.com/",
}
apply_auth(headers, auth)
# 4. Fire all searches through a single persistent session.
# 3. Fire all searches through a single persistent session, sharing the
# auth-bearing headers.
sem = asyncio.Semaphore(MAX_CONCURRENT)
connector = aiohttp.TCPConnector(
limit=MAX_CONCURRENT,
keepalive_timeout=30,
)
async with aiohttp.ClientSession(
timeout=timeout,
connector=connector,
connector=aiohttp.TCPConnector(limit=MAX_CONCURRENT, keepalive_timeout=30),
) as session:
ctx = _SearchCtx(
session=session,
search_url=search_url,
headers=headers,
headers=_search_headers(auth),
cache=cache,
polls=polls,
extras=extras,
@ -168,7 +182,7 @@ async def _fetch_batch(
extras = _HLTBExtras()
search_results = await _search_batch(games, cache, polls, progress_cb, extras)
# 5. Fetch leisure times + DLC from game detail pages.
# 4. Fetch leisure times + DLC from game detail pages.
logger.info(
"Fetching leisure times for %d games from detail pages...",
len(search_results),

View File

@ -144,7 +144,7 @@ def restore_prior_hours(refreshed: dict[int, float], prior: dict[int, float]) ->
we already had -- prior leisure/DLC figures are trusted over a miss.
"""
for app_id, old_hours in prior.items():
if old_hours > 0 and refreshed.get(app_id, -1.0) <= 0:
if refreshed.get(app_id, -1.0) <= 0 < old_hours:
refreshed[app_id] = old_hours

View File

@ -130,8 +130,6 @@ class LeisureKiller:
) -> None:
"""Signal this tick's leisure targets if the leisure budget is spent.
Never raises: the enforce loop has no per-tick guard.
Args:
state: Accounting state after this tick's policy.
rules: Policy for this tick (``leisure`` set by the billing step).
@ -140,7 +138,9 @@ class LeisureKiller:
"""
try:
self._enforce(state, rules, verdict=verdict, stamp=stamp)
except Exception as exc: # the loop must outlive any kill bug
# Never raises: the enforce loop has no per-tick guard and must outlive
# any kill bug, so this catch-all is on purpose.
except Exception as exc: # pylint: disable=broad-exception-caught
message = f"{type(exc).__name__}: {exc}"
if message != self._last_error:
self._last_error = message
@ -179,6 +179,12 @@ class LeisureKiller:
"day_key": state.day_key,
"seconds": round(state.seconds, 3),
}
self._escalate(live, base, stamp)
def _escalate(
self, live: dict[_Ident, str], base: dict[str, object], stamp: float
) -> None:
"""SIGTERM each live target once, then SIGKILL it after the grace."""
for ident, key in sorted(live.items()):
note = {"key": key, "name": exe_name(ident[0]), **base}
first = self._termed.get(ident)

View File

@ -120,7 +120,8 @@ def observe_shadow(owners: dict[int, str], *, stamp: float) -> ShadowVerdict:
if paused is not None:
return ShadowVerdict(paused_by=paused, keys=())
fact = read_now(NOW_PATH, stamp, max_age=_MAX_AGE_SECONDS)
except Exception as exc: # the loop must outlive any shadow input bug
# Catch-all on purpose: the loop must outlive any shadow input bug.
except Exception as exc: # pylint: disable=broad-exception-caught
if _should_warn(exc):
logger.warning(
"Shadow inputs unreadable (%r); failing closed.", exc, exc_info=True

View File

@ -125,12 +125,46 @@ def _print_player_speed_scenario(
_echo(f" @ {daily:.0f} h/day → {estimate}")
def _print_header(
qualified: list[_GameTimes], skips: list[int], state: State, games_done: int
) -> None:
"""Print the banner, the filter counts and the excluded games."""
hltb_skip, linux_skip, no_data_skip = skips
total_q = len(qualified)
rule = "═" * 70
_echo(f"\n{rule}\n BACKLOG COMPLETION ESTIMATES\n{rule}")
_echo(f"\n Qualifying games: {total_q}")
if hltb_skip:
_echo(f" HLTB-skipped: {hltb_skip} (confidence too low)")
if linux_skip:
_echo(f" Linux-skipped: {linux_skip} (poor ProtonDB rating)")
if no_data_skip:
_echo(f" No-data-skipped: {no_data_skip} (no HLTB hours at all)")
missing_rush_final = sum(1 for e in qualified if e.rush_hours <= 0)
if missing_rush_final:
_echo(
f"\n Note: {missing_rush_final}/{total_q} games still missing"
" rush/leisure data (HLTB search may not have matched them)."
)
elif total_q:
_echo(
f"\n Detail data: rush + leisure available for all {total_q}"
" qualifying games."
)
if state.current_app_id:
_echo(
f"\n Current game: {state.current_game_name} (excluded from totals)"
)
_echo(f" Finished games: {games_done} (excluded from totals)")
def cmd_stats(_config: Config, state: State) -> None:
"""Display backlog completion-time statistics.
Filters games by the same HLTB-confidence and Linux-compatibility rules
used when picking the next game. Auto-fetches missing rush/leisure detail
data before printing. Shows five scenarios:
Filters games by the HLTB-confidence and Linux-compatibility rules used to
pick the next game, auto-fetching missing rush/leisure data. Shows:
1. At your current pace (games finished per day since enforcement started).
2. Rush — avg comp_100 + DLC completion time per HLTB.
@ -138,8 +172,8 @@ def cmd_stats(_config: Config, state: State) -> None:
4. Worst — absolute maximum recorded time (any category) per HLTB.
5. Your play style — extrapolated from completed-game calibration vs HLTB.
Ends with one line per kind of HLTB failure (the cache is kept for those
games) and exits 1 when some HLTB batch refreshed nothing at all.
Ends with one line per HLTB failure kind (their cache is kept) and exits 1
when some HLTB batch refreshed nothing at all.
"""
reset_failures()
try:
@ -170,44 +204,12 @@ def _print_stats(_config: Config, state: State) -> None:
# Ensure completed games have rush/leisure data for pace calibration.
_ensure_completed_rush_data(games)
qualified, hltb_skip, linux_skip, no_data_skip = _filter_qualifying_games(
games, state
)
qualified, *skips = _filter_qualifying_games(games, state)
if _ensure_rush_data(qualified):
# Re-filter picks up updated rush/leisure caches; ProtonDB is now cached.
qualified, hltb_skip, linux_skip, no_data_skip = _filter_qualifying_games(
games, state
)
qualified, *skips = _filter_qualifying_games(games, state)
total_q = len(qualified)
_echo(f"\n{'═' * 70}")
_echo(" BACKLOG COMPLETION ESTIMATES")
_echo(f"{'═' * 70}")
_echo(f"\n Qualifying games: {total_q}")
if hltb_skip:
_echo(f" HLTB-skipped: {hltb_skip} (confidence too low)")
if linux_skip:
_echo(f" Linux-skipped: {linux_skip} (poor ProtonDB rating)")
if no_data_skip:
_echo(f" No-data-skipped: {no_data_skip} (no HLTB hours at all)")
missing_rush_final = sum(1 for e in qualified if e.rush_hours <= 0)
if missing_rush_final:
_echo(
f"\n Note: {missing_rush_final}/{total_q} games still missing"
" rush/leisure data (HLTB search may not have matched them)."
)
elif total_q:
_echo(
f"\n Detail data: rush + leisure available for all {total_q}"
" qualifying games."
)
if state.current_app_id:
_echo(
f"\n Current game: {state.current_game_name} (excluded from totals)"
)
_echo(f" Finished games: {games_done} (excluded from totals)")
_print_header(qualified, skips, state, games_done)
_echo(f"\n{_LINE}")
_print_pace_scenario(state, total_q, games_done_since_start)
@ -242,8 +244,7 @@ def _print_stats(_config: Config, state: State) -> None:
_print_worst_example(qualified)
# Pace calibration uses the freshly-updated cache (both fetches above ran).
raw_cache = _read_raw_cache()
pace_vs_hltb = compute_pace_vs_hltb(games, raw_cache)
pace_vs_hltb = compute_pace_vs_hltb(games, _read_raw_cache())
_print_player_speed_scenario(pace_vs_hltb, rush_total, leisure_total)
_echo(f"\n{_LINE}\n")

View File

@ -7,6 +7,7 @@ the 250-line cap. Leaf helpers: nothing here calls back into ``_stats``.
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
import logging
from typing import TYPE_CHECKING
@ -41,6 +42,46 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class _HltbCaches:
"""The HLTB caches the stats filter reads, loaded once per filter run."""
rush: dict[int, float]
leisure_100h: dict[int, float]
game_id: dict[int, int]
hours: dict[int, float]
@classmethod
def load(cls) -> _HltbCaches:
"""Read every cache from disk."""
return cls(
rush=load_hltb_rush_cache(),
leisure_100h=load_hltb_leisure_100h_cache(),
game_id=load_hltb_game_id_cache(),
hours=load_hltb_cache(),
)
def times_for(self, game: GameInfo) -> _GameTimes | None:
"""*game*'s time estimates, or None when HLTB has no time for it."""
rush = self.rush.get(game.app_id, -1)
leisure = self.leisure_100h.get(game.app_id, -1)
# worst_hours = max of: snapshot completionist, HLTB hours cache (fallback
# when snapshot is stale/missing), and leisure_100h (slowest 100% time).
snap_hours = game.completionist_hours if game.completionist_hours > 0 else -1
cache_hours = self.hours.get(game.app_id, -1)
worst_candidates = [v for v in (snap_hours, cache_hours, leisure) if v > 0]
worst = max(worst_candidates) if worst_candidates else -1
if worst <= 0 and rush <= 0 and leisure <= 0:
return None
return _GameTimes(
game=game,
worst_hours=worst,
rush_hours=rush,
leisure_100h=leisure,
hltb_game_id=self.game_id.get(game.app_id, 0),
)
def _filter_qualifying_games(
games: list[GameInfo],
state: State,
@ -53,10 +94,7 @@ def _filter_qualifying_games(
Returns:
(qualified_list, hltb_skipped, linux_skipped, no_data_skipped)
"""
rush_cache = load_hltb_rush_cache()
leisure_100h_cache = load_hltb_leisure_100h_cache()
game_id_cache = load_hltb_game_id_cache()
hours_cache = load_hltb_cache()
caches = _HltbCaches.load()
exclude = set(state.finished_app_ids)
if state.current_app_id is not None:
@ -66,53 +104,25 @@ def _filter_qualifying_games(
_apply_cached_confidence_to_candidates(candidates)
_refresh_candidate_confidence_batch(candidates)
hltb_skipped = 0
linux_skipped = 0
no_data_skipped = 0
app_ids_to_check: list[int] = []
conf_ok: list[GameInfo] = []
for game in candidates:
if _confidence_fail_reasons(game):
hltb_skipped += 1
continue
conf_ok.append(game)
app_ids_to_check.append(game.app_id)
conf_ok = [game for game in candidates if not _confidence_fail_reasons(game)]
hltb_skipped = len(candidates) - len(conf_ok)
ratings: dict[int, ProtonDBRating] = {}
if app_ids_to_check:
ratings = fetch_protondb_ratings(app_ids_to_check)
if conf_ok:
ratings = fetch_protondb_ratings([game.app_id for game in conf_ok])
linux_skipped = 0
no_data_skipped = 0
qualified: list[_GameTimes] = []
for game in conf_ok:
rating = ratings.get(game.app_id, ProtonDBRating(app_id=game.app_id))
if not rating.is_playable:
if not ratings.get(game.app_id, ProtonDBRating(app_id=game.app_id)).is_playable:
linux_skipped += 1
continue
rush = rush_cache.get(game.app_id, -1)
leisure = leisure_100h_cache.get(game.app_id, -1)
# worst_hours = max of: snapshot completionist, HLTB hours cache (fallback
# when snapshot is stale/missing), and leisure_100h (slowest 100% time).
snap_hours = game.completionist_hours if game.completionist_hours > 0 else -1
cache_hours = hours_cache.get(game.app_id, -1)
worst_candidates = [v for v in (snap_hours, cache_hours, leisure) if v > 0]
worst = max(worst_candidates) if worst_candidates else -1
if worst <= 0 and rush <= 0 and leisure <= 0:
times = caches.times_for(game)
if times is None:
no_data_skipped += 1
continue
qualified.append(
_GameTimes(
game=game,
worst_hours=worst,
rush_hours=rush,
leisure_100h=leisure,
hltb_game_id=game_id_cache.get(game.app_id, 0),
)
)
qualified.append(times)
return qualified, hltb_skipped, linux_skipped, no_data_skipped

View File

@ -46,10 +46,7 @@ class GameInfo:
@property
def is_complete(self) -> bool:
"""True if all achievements are unlocked."""
return (
self.total_achievements > 0
and self.unlocked_achievements >= self.total_achievements
)
return 0 < self.total_achievements <= self.unlocked_achievements
def to_snapshot(self) -> dict[str, Any]:
"""Serialize to JSON-safe dict."""

View File

@ -84,7 +84,7 @@ def resolve(method: str, path: str) -> tuple[Route, tuple[str, ...]] | str | Non
"""
path_known = False
for entry in ROUTES:
match = entry.pattern.fullmatch(path)
match = re.fullmatch(entry.pattern, path)
if match is None:
continue
if entry.method == method:

View File

@ -101,7 +101,8 @@ class _Handler(BaseHTTPRequestHandler):
self._dispatch(method, split.path, _query(split.query))
except ApiError as exc:
send_error(self, exc)
except Exception:
# Catch-all on purpose: any bug answers 500, the server keeps serving.
except Exception: # pylint: disable=broad-exception-caught
logger.exception("%s %s failed", method, split.path)
msg = "The server failed to answer this request; see its log."
send_error(

View File

@ -14,6 +14,7 @@ The server only reads files here, so a job is unaffected by any of this.
from __future__ import annotations
import contextlib
from http import HTTPStatus
import json
import time
@ -106,9 +107,8 @@ def stream_job_events(handler: BaseHTTPRequestHandler, request: Request) -> None
handler.send_header("Cache-Control", "no-store")
handler.send_header("X-Accel-Buffering", "no")
handler.end_headers()
try:
# The browser going away is not an error; it will reconnect if it cares.
with contextlib.suppress(BrokenPipeError, ConnectionResetError):
handler.wfile.write(f"retry: {_RETRY_MS}\n\n".encode())
_pump(handler, job_id, after)
handler.wfile.flush()
except BrokenPipeError, ConnectionResetError:
return # The browser went away; it will reconnect if it cares.

View File

@ -163,8 +163,11 @@ def _run_command(
# The gate already printed its banner (now log events); the
# summary is the one-line form of the same refusal.
sink.flush()
reason = lock_reason(command, config, State.load())
return _failed(reason or "Refused by the lock gate.", EXIT_REFUSED)
return _failed(
lock_reason(command, config, State.load())
or "Refused by the lock gate.",
EXIT_REFUSED,
)
try:
data = job(params, progress, prompter)
except SystemExit as exc:
@ -205,10 +208,12 @@ def run_job(job_dir: Path) -> int:
outcome = _failed(exc.message, EXIT_REFUSED)
except PrivilegedViaDaemonError as exc:
outcome = _failed(str(exc), EXIT_REFUSED)
except Exception as exc:
# Catch-all on purpose: every job ends with a result, however it fails.
except Exception as exc: # pylint: disable=broad-exception-caught
logger.exception("Job %s crashed", job_dir.name)
outcome = _failed(f"{type(exc).__name__}: {exc}")
except BaseException as exc: # SystemExit from outside a handler, etc.
# SystemExit from outside a handler, etc.: still write the result.
except BaseException as exc: # pylint: disable=broad-exception-caught
logger.exception("Job %s stopped", job_dir.name)
outcome = _failed(f"{type(exc).__name__}: {exc}")
_finish(job_dir, writer, outcome)

View File

@ -18,7 +18,7 @@ import shutil
import subprocess
import sys
import time
from typing import Final
from typing import BinaryIO, Final
from steam_backlog_enforcer.config import _atomic_write
from steam_backlog_enforcer.jobs._errors import JobError
@ -111,6 +111,25 @@ def _confirm_started(proc: subprocess.Popen[bytes], path: Path) -> None:
time.sleep(_POLL_SECONDS)
def _start_session(
path: Path, out: BinaryIO, env: dict[str, str], lock_fd: int | None
) -> subprocess.Popen[bytes]:
"""Start the job process in its own session; the caller owns it.
No ``with``: the job outlives the CLI call that spawned it.
"""
return subprocess.Popen(
_spawn_argv(path),
stdin=subprocess.DEVNULL,
stdout=out,
stderr=out,
cwd=_PACKAGE_ROOT,
env=env,
start_new_session=True,
pass_fds=() if lock_fd is None else (lock_fd,),
)
def spawn_job(path: Path, lock_fd: int | None) -> int:
"""Start the detached job process for *path* and return its pid.
@ -130,16 +149,7 @@ def spawn_job(path: Path, lock_fd: int | None) -> int:
env[LOCK_FD_ENV] = str(lock_fd)
try:
with (path / OUTPUT_NAME).open("ab") as out:
proc = subprocess.Popen(
_spawn_argv(path),
stdin=subprocess.DEVNULL,
stdout=out,
stderr=out,
cwd=_PACKAGE_ROOT,
env=env,
start_new_session=True,
pass_fds=() if lock_fd is None else (lock_fd,),
)
proc = _start_session(path, out, env, lock_fd)
except OSError as exc:
msg = f"The job process could not be started: {exc}."
raise _fail_unstarted(path, msg) from exc

View File

@ -21,7 +21,7 @@ import logging
from typing import TYPE_CHECKING
from steam_backlog_enforcer._achievement_skip import zero_achievement_ids
from steam_backlog_enforcer._actions import allowed_app_ids
from steam_backlog_enforcer._actions import allowed_app_ids as _allowed_ids_of
from steam_backlog_enforcer._cdp import _cdp_result_value, _evaluate_js
from steam_backlog_enforcer._echo import _echo
from steam_backlog_enforcer._owned_apps_cache import get_all_owned_app_ids
@ -233,7 +233,7 @@ def hide_others_and_report(config: Config, state: State, *, lead: str = "\n ")
owned_ids = get_all_owned_app_ids(config)
if not owned_ids:
return
hidden, skipped = try_hide_other_games(owned_ids, allowed_app_ids(state))
hidden, skipped = try_hide_other_games(owned_ids, _allowed_ids_of(state))
if skipped is not None:
_echo(f"{lead}Library hiding: skipped ({skipped})")
elif hidden > 0:

View File

@ -78,11 +78,8 @@ def do_scan(config: Config, state: State) -> list[GameInfo]:
client = SteamAPIClient(config.steam_api_key, config.steam_id)
start = time.time()
done_count = 0
def progress(current: int, total: int) -> None:
nonlocal done_count
done_count = current
if current % 50 == 0 or current == total:
_echo(f"\r Scanning achievements: {current}/{total}", end="", flush=True)
@ -115,19 +112,19 @@ def do_scan(config: Config, state: State) -> list[GameInfo]:
polls_cache = load_hltb_polls_cache()
count_comp_cache = load_hltb_count_comp_cache()
for g in games:
hours = hltb_cache.get(g.app_id, -1)
g.completionist_hours = hours
g.completionist_hours = hltb_cache.get(g.app_id, -1)
g.comp_100_count = polls_cache.get(g.app_id, 0)
g.count_comp = count_comp_cache.get(g.app_id, 0)
found = sum(1 for h in hltb_cache.values() if h > 0)
_echo(f" HLTB data: {found} games have completion estimates")
_echo(
f" HLTB data: {sum(1 for h in hltb_cache.values() if h > 0)} games"
" have completion estimates"
)
# Save snapshot.
save_snapshot([g.to_snapshot() for g in games])
complete = [g for g in games if g.is_complete]
incomplete_games = [g for g in games if not g.is_complete]
_echo(f"\nResults: {len(complete)} complete, {len(incomplete_games)} incomplete")
n_complete = sum(1 for g in games if g.is_complete)
_echo(f"\nResults: {n_complete} complete, {len(games) - n_complete} incomplete")
# Auto-pick a game if none assigned.
if state.current_app_id is None:

View File

@ -37,6 +37,9 @@ from steam_backlog_enforcer.config import (
logger = logging.getLogger(__name__)
# The install script rewrites /etc/hosts and re-applies its guards.
_HOSTS_INSTALL_TIMEOUT_SECONDS = 120
def is_store_blocked() -> bool:
"""Check if Steam Store domains are blocked in /etc/hosts or iptables."""
@ -110,7 +113,7 @@ def _block_via_hosts_install() -> bool:
[SUDO, BASH, str(HOSTS_INSTALL_SCRIPT), "--no-flush-dns"],
capture_output=True,
text=True,
timeout=120,
timeout=_HOSTS_INSTALL_TIMEOUT_SECONDS,
check=False,
)
except OSError, subprocess.SubprocessError: