steam-backlog-enforcer/steam_backlog_enforcer/_hltb_detail.py
Krzysztof kuhy Rudnicki 57d262b711
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (mypy, mypy, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (pylint, pylint, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (rest, , mypy,pylint) (push) Waiting to run
Tests / test (push) Waiting to run
Tests / web-coverage (push) Waiting to run
repo-contract / repo contract (push) Waiting to run
Pylint gate 10 over the whole package
Raise the pylint hook to --fail-under=10.0 and make it lint the whole
package on every run (pass_filenames: false): over the staged subset a
9.99 tree passed at commit time.

Ports de2aea3 (9b9342ba-1) onto main, minus its web/ and requirements.txt
bumps and the iptables dedupe f1822f1 already landed; _hltb_confidence and
_hltb_search keep main's 429 handling. _search_batch splits out
_acquire_auth / _search_headers, and _stats.py is back to 250 lines.

Drops the min-similarity-lines = 8 floor (0 duplicate-code hits at the
default) and sets max-attributes = 15 for the flat persisted records, as
kuhy decided. Tests stay outside pylint, as at every site already at 10.

Todo: fda71ed1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 16:57:35 +01:00

231 lines
7.7 KiB
Python

"""Detail page parsing and leisure time / DLC fetching for HLTB."""
from __future__ import annotations
import asyncio
import logging
from typing import Any
import aiohttp
from steam_backlog_enforcer._hltb_comp_extract import (
_collect_dlc_relationships,
_extract_comp_100_avg_and_high,
_extract_dlc_relationships,
_fetch_detail_one,
_platform_comp_high_candidates,
)
from steam_backlog_enforcer._hltb_page_parse import (
_apply_dlc_leisure_overrides,
_as_positive_int,
)
from steam_backlog_enforcer._hltb_types import (
_SAVE_INTERVAL,
MAX_CONCURRENT,
HLTBResult,
ProgressCb,
_HLTBExtras,
save_hltb_cache,
)
from steam_backlog_enforcer._progress import gather_tracked
logger = logging.getLogger(__name__)
def _extract_base_leisure_hours(game_data: dict[str, Any]) -> float:
"""Extract base-game leisure hours from game detail data.
Returns the highest (slowest) time to beat across all play styles.
Candidates considered:
1. ``comp_high`` from each entry in ``platformData`` — the per-platform
slowest individual submission displayed on the HLTB page.
2. The ``_h`` (leisure/high) fields from ``game[0]``:
``comp_main_h``, ``comp_plus_h``, ``comp_100_h``, ``comp_all_h``.
3. Falls back to average times: ``comp_main``, ``comp_plus``, ``comp_100``.
"""
games = game_data.get("game", [])
if not isinstance(games, list) or not games:
return -1
if not isinstance(games[0], dict):
return -1
base = games[0]
candidates = _platform_comp_high_candidates(game_data)
# 2. Leisure/high fields from the game record
for field in ("comp_main_h", "comp_plus_h", "comp_100_h", "comp_all_h"):
v = _as_positive_int(base.get(field, 0))
if v > 0:
candidates.append(v)
leisure_s = max(candidates) if candidates else 0
# 3. Fallback: average completion times
if leisure_s <= 0:
avg_candidates = [
_as_positive_int(base.get("comp_main", 0)),
_as_positive_int(base.get("comp_plus", 0)),
_as_positive_int(base.get("comp_100", 0)),
]
leisure_s = max(avg_candidates)
if leisure_s <= 0:
return -1
return round(leisure_s / 3600, 2)
def _extract_leisure_hours(game_data: dict[str, Any]) -> float:
"""Compute total leisure hours: base game + all DLCs.
Uses the highest (slowest) time across ``platformData comp_high`` and
leisure ``_h`` fields from ``game[0]``. Falls back to average completion
times. Also sums leisure time from any DLC listed in ``relationships``.
"""
base_hours = _extract_base_leisure_hours(game_data)
if base_hours <= 0:
return -1
total_hours = base_hours
# Add DLC leisure times from relationships.
for _dlc_id, fallback_hours in _extract_dlc_relationships(game_data):
total_hours += fallback_hours
return round(total_hours, 2)
def _process_game_detail(
game_data: dict[str, Any],
dlc_rels: list[tuple[int, float]],
dlc_hours_by_id: dict[int, float],
) -> tuple[float, float, float]:
"""Return (leisure_hours, rush_hours, leisure_100h) for one game's detail data."""
leisure = _extract_leisure_hours(game_data)
if leisure > 0:
leisure = _apply_dlc_leisure_overrides(leisure, dlc_rels, dlc_hours_by_id)
avg_h, high_h = _extract_comp_100_avg_and_high(game_data)
rush_h = -1.0
if avg_h > 0:
dlc_rush = sum(fh for _, fh in dlc_rels if fh > 0)
rush_h = round(avg_h + dlc_rush, 2)
l100 = -1.0
if high_h > 0:
l100 = _apply_dlc_leisure_overrides(high_h, dlc_rels, dlc_hours_by_id)
return leisure, rush_h, l100
def _apply_detail_to_extras(
app_id: int,
game_data: dict[str, Any],
dlc_rels: list[tuple[int, float]],
dlc_hours_by_id: dict[int, float],
extras: _HLTBExtras,
) -> float:
"""Update extras in-place from detail data; return leisure hours (or -1)."""
leisure, rush_h, l100 = _process_game_detail(game_data, dlc_rels, dlc_hours_by_id)
if rush_h > 0:
extras.rush[app_id] = rush_h
if l100 > 0:
extras.leisure_100h[app_id] = l100
# The search API sometimes returns count_comp=0 even when the detail page
# has all-playstyles completion counts. Use the detail value when present.
games_list = game_data.get("game")
if isinstance(games_list, list) and games_list:
count_comp_detail = _as_positive_int(games_list[0].get("count_comp", 0))
if count_comp_detail > 0:
extras.count_comp[app_id] = count_comp_detail
return leisure
async def _fetch_all_details(
valid: list[HLTBResult],
) -> tuple[
list[dict[str, Any] | None], dict[int, list[tuple[int, float]]], dict[int, float]
]:
"""Fetch every detail page, then the leisure hours of every DLC they list."""
timeout = aiohttp.ClientTimeout(total=30, sock_read=20)
sem = asyncio.Semaphore(MAX_CONCURRENT)
connector = aiohttp.TCPConnector(limit=MAX_CONCURRENT, keepalive_timeout=30)
async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
coros = [_fetch_detail_one(sem, session, r.hltb_game_id) for r in valid]
names = [r.game_name for r in valid]
details = await gather_tracked("Fetching HLTB detail pages", coros, names)
dlc_relationships_by_app, dlc_ids = _collect_dlc_relationships(valid, details)
return (
details,
dlc_relationships_by_app,
await _fetch_dlc_leisure_hours(sem, session, dlc_ids),
)
async def _fetch_leisure_times(
search_results: list[HLTBResult],
cache: dict[int, float],
polls: dict[int, int],
progress_cb: ProgressCb | None,
extras: _HLTBExtras | None = None,
) -> None:
"""Fetch leisure times from game detail pages for all search results.
Updates ``cache`` in-place with leisure hours (including DLC time).
Also populates ``extras.rush`` (avg comp_100 + DLC) and
``extras.leisure_100h`` (comp_100_h + DLC leisure).
The ``polls`` and ``extras.count_comp`` are forwarded to
:func:`save_hltb_cache` so confidence metrics persist.
"""
if extras is None:
extras = _HLTBExtras()
valid = [r for r in search_results if r.hltb_game_id > 0]
if not valid:
return
details, dlc_relationships_by_app, dlc_hours_by_id = await _fetch_all_details(valid)
found = 0
for done, (r, game_data) in enumerate(zip(valid, details, strict=False), 1):
if game_data is not None:
leisure = _apply_detail_to_extras(
r.app_id,
game_data,
dlc_relationships_by_app.get(r.app_id, []),
dlc_hours_by_id,
extras,
)
if leisure > 0:
r.completionist_hours = leisure
cache[r.app_id] = leisure
found += 1
if progress_cb is not None:
progress_cb(done, len(valid), found, r.game_name)
if not done % _SAVE_INTERVAL:
save_hltb_cache(cache, polls, extras)
async def _fetch_dlc_leisure_hours(
sem: asyncio.Semaphore,
session: aiohttp.ClientSession,
dlc_ids: list[int],
) -> dict[int, float]:
"""Fetch leisure hours for each DLC game id."""
if not dlc_ids:
return {}
coros = [_fetch_detail_one(sem, session, dlc_id) for dlc_id in dlc_ids]
dlc_details = await gather_tracked("Fetching HLTB DLC pages", coros)
dlc_hours_by_id: dict[int, float] = {}
for dlc_id, dlc_data in zip(dlc_ids, dlc_details, strict=False):
if dlc_data is None:
continue
dlc_leisure = _extract_base_leisure_hours(dlc_data)
if dlc_leisure > 0:
dlc_hours_by_id[dlc_id] = dlc_leisure
return dlc_hours_by_id