mirror of
https://github.com/kuhyx/steam-backlog-enforcer.git
synced 2026-11-10 18:07:07 +01:00
HLTB rate limit (429) wipes confidence data in hl…
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (mypy, mypy, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (pylint, pylint, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (rest, , mypy,pylint) (push) Waiting to run
Tests / test (push) Waiting to run
Tests / web-coverage (push) Waiting to run
repo-contract / repo contract (push) Waiting to run
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (mypy, mypy, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (pylint, pylint, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (rest, , mypy,pylint) (push) Waiting to run
Tests / test (push) Waiting to run
Tests / web-coverage (push) Waiting to run
repo-contract / repo contract (push) Waiting to run
Todo: 9b9342ba
This commit is contained in:
parent
2ec53d977d
commit
8cc31a2c87
@ -4,7 +4,7 @@
|
||||
aiohttp==3.14.5
|
||||
bandit==1.9.4
|
||||
codespell==2.4.3
|
||||
coverage==7.17.0
|
||||
coverage==7.17.2
|
||||
earned-time @ git+https://github.com/kuhyx/utils@earned-time-v0.21.0#subdirectory=earned_time
|
||||
howlongtobeatpy==1.0.24
|
||||
leisure-watch @ git+https://github.com/kuhyx/leisure-watch@leisure-watch-v0.2.7
|
||||
@ -14,11 +14,11 @@ pre-commit==4.7.0
|
||||
pylint==4.1.3
|
||||
pytest==9.2.0
|
||||
pytest-cov==7.1.0
|
||||
pytest-randomly==5.0.0
|
||||
pytest-randomly==5.1.0
|
||||
pytest-sugar==1.1.1
|
||||
pytest-timeout==2.4.0
|
||||
pytest-xdist==3.8.0
|
||||
requests==2.34.2
|
||||
ruff==0.17.3
|
||||
ruff==0.17.4
|
||||
types-requests==2.33.0.20261030
|
||||
websockets==17.4
|
||||
|
||||
62
steam_backlog_enforcer/_hltb_auth_refresh.py
Normal file
62
steam_backlog_enforcer/_hltb_auth_refresh.py
Normal file
@ -0,0 +1,62 @@
|
||||
"""Renew the HLTB auth token during a long, paced search batch.
|
||||
|
||||
The token from ``/api/search/site/init`` stops working a few minutes after it
|
||||
is issued: on 2026-11-08 every search ~3 min in came back 403 while a fresh
|
||||
token got 200 for the same game. Unpaced batches finished before that; a
|
||||
paced ~800-game refresh takes 15+ min, so the token is renewed by age.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from steam_backlog_enforcer._hltb_pacing import pause_after_request
|
||||
from steam_backlog_enforcer._hltb_search_api import _get_auth_info
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from steam_backlog_enforcer._hltb_search_one import _SearchCtx
|
||||
from steam_backlog_enforcer._hltb_types import _AuthInfo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Well inside the observed ~3 min lifetime (still valid at ~146 s).
|
||||
AUTH_MAX_AGE_S = 90.0
|
||||
# Indirection so tests can drive a fake clock.
|
||||
_now = time.monotonic
|
||||
|
||||
|
||||
def apply_auth(headers: dict[str, str], auth: _AuthInfo) -> None:
|
||||
"""Put *auth* into the search request headers, replacing any old values."""
|
||||
headers["x-auth-token"] = auth.token
|
||||
if auth.hp_key:
|
||||
headers["x-hp-key"] = auth.hp_key
|
||||
headers["x-hp-val"] = auth.hp_val
|
||||
else:
|
||||
headers.pop("x-hp-key", None)
|
||||
headers.pop("x-hp-val", None)
|
||||
|
||||
|
||||
async def ensure_fresh_auth(ctx: _SearchCtx) -> str | None:
|
||||
"""Renew ``ctx.auth`` once it is too old.
|
||||
|
||||
Returns ``None`` when the auth is usable, else the failure reason. A
|
||||
failed renewal trips the breaker: the stale token is never used again.
|
||||
"""
|
||||
if _now() - ctx.auth_at < AUTH_MAX_AGE_S:
|
||||
return None
|
||||
async with ctx.auth_lock:
|
||||
if _now() - ctx.auth_at < AUTH_MAX_AGE_S:
|
||||
return None # another lookup renewed it while we waited
|
||||
failures: list[str] = []
|
||||
auth = await _get_auth_info(ctx.search_url, ctx.session, failures)
|
||||
await pause_after_request()
|
||||
if auth is None:
|
||||
ctx.breaker.tripped = True
|
||||
return f"auth refresh {failures[0] if failures else 'no token'}"
|
||||
ctx.auth = auth
|
||||
apply_auth(ctx.headers, auth)
|
||||
ctx.auth_at = _now()
|
||||
logger.info("HLTB auth token refreshed.")
|
||||
return None
|
||||
@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
|
||||
from steam_backlog_enforcer._hltb_types import (
|
||||
ProgressCb,
|
||||
_HLTBExtras,
|
||||
@ -83,7 +84,11 @@ def fetch_hltb_times_cached(
|
||||
report = reporting_progress_cb(
|
||||
"Fetching HLTB times", len(uncached), progress_cb
|
||||
)
|
||||
try:
|
||||
elapsed = fetch_hltb_times_timed(uncached, cache, polls, report, extras)
|
||||
except HLTBUnavailableError:
|
||||
# Already logged and tallied by the batch; the cache is untouched.
|
||||
return cache
|
||||
|
||||
# Final save.
|
||||
save_hltb_cache(cache, polls, extras)
|
||||
|
||||
@ -13,6 +13,13 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import aiohttp
|
||||
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
detail_status_failed,
|
||||
exc_reason,
|
||||
record_failure,
|
||||
status_reason,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_pacing import pause_after_request
|
||||
from steam_backlog_enforcer._hltb_page_parse import (
|
||||
_as_positive_int,
|
||||
_parse_game_page,
|
||||
@ -110,10 +117,16 @@ async def _fetch_detail_one(
|
||||
if resp.status == HTTPStatus.OK:
|
||||
html = await resp.text()
|
||||
return _parse_game_page(html)
|
||||
if not detail_status_failed(resp.status):
|
||||
return None # no such page: a real answer, not a failure
|
||||
reason = status_reason(resp.status)
|
||||
except (TimeoutError, aiohttp.ClientError) as exc:
|
||||
logger.debug(
|
||||
"HLTB detail fetch failed for game_id=%d: %s",
|
||||
hltb_game_id,
|
||||
exc,
|
||||
reason = exc_reason(exc)
|
||||
finally:
|
||||
await pause_after_request()
|
||||
# The caller keeps the cached hours when this returns None.
|
||||
logger.warning(
|
||||
"HLTB detail fetch failed for game_id=%d (%s)", hltb_game_id, reason
|
||||
)
|
||||
record_failure(f"detail {reason}")
|
||||
return None
|
||||
|
||||
@ -10,6 +10,7 @@ import logging
|
||||
import time
|
||||
|
||||
from steam_backlog_enforcer._hltb_cached import reporting_progress_cb
|
||||
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
|
||||
from steam_backlog_enforcer._hltb_types import (
|
||||
HLTB_BASE_URL,
|
||||
ProgressCb,
|
||||
@ -38,28 +39,36 @@ logger = logging.getLogger(__name__)
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def refetch_poll_counts(missing: list[tuple[int, str]]) -> dict[int, int]:
|
||||
"""Re-search ``missing`` for poll counts and return the refreshed polls cache.
|
||||
def refresh_confidence(
|
||||
games: list[tuple[int, str]],
|
||||
) -> tuple[dict[int, float], dict[int, int], dict[int, int]]:
|
||||
"""Re-search ``games`` (cached or not) and save hours, polls and count_comp.
|
||||
|
||||
The search-level fetch skips anything already in the hours cache, so the
|
||||
hours entries are dropped first to force it -- then put back where the
|
||||
refetch found nothing better, since a prior leisure/DLC estimate is
|
||||
trusted over a miss.
|
||||
Nothing is dropped from the cache first: a failed lookup writes nothing,
|
||||
so the old entry survives. Prior hours beat a clean miss, since a prior
|
||||
leisure/DLC estimate is trusted over "not found". When HLTB is unavailable
|
||||
the cache is returned unchanged and nothing is saved.
|
||||
|
||||
Returns:
|
||||
The (hours, polls, count_comp) caches after the refresh.
|
||||
"""
|
||||
cache = load_hltb_cache()
|
||||
polls = load_hltb_polls_cache()
|
||||
preserved_hours = {aid: cache[aid] for aid, _ in missing if aid in cache}
|
||||
for aid, _name in missing:
|
||||
cache.pop(aid, None)
|
||||
save_hltb_cache(cache, polls)
|
||||
count_comp = load_hltb_count_comp_cache()
|
||||
prior_hours = {aid: cache[aid] for aid, _ in games if aid in cache}
|
||||
try:
|
||||
fetch_hltb_confidence(games, cache=cache, polls=polls, count_comp=count_comp)
|
||||
except HLTBUnavailableError:
|
||||
# Already logged and tallied by the batch; the cache is untouched.
|
||||
return cache, polls, count_comp
|
||||
restore_prior_hours(cache, prior_hours)
|
||||
save_hltb_cache(cache, polls, _HLTBExtras(count_comp=count_comp))
|
||||
return cache, polls, count_comp
|
||||
|
||||
fetch_hltb_confidence_cached(missing)
|
||||
|
||||
refreshed_hours = load_hltb_cache()
|
||||
refreshed_polls = load_hltb_polls_cache()
|
||||
restore_prior_hours(refreshed_hours, preserved_hours)
|
||||
save_hltb_cache(refreshed_hours, refreshed_polls)
|
||||
return refreshed_polls
|
||||
def refetch_poll_counts(missing: list[tuple[int, str]]) -> dict[int, int]:
|
||||
"""Re-search ``missing`` for poll counts and return the refreshed polls cache."""
|
||||
return refresh_confidence(missing)[1]
|
||||
|
||||
|
||||
def fetch_hltb_confidence_cached(
|
||||
@ -79,6 +88,7 @@ def fetch_hltb_confidence_cached(
|
||||
len(games) - len(uncached),
|
||||
)
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
fetch_hltb_confidence(
|
||||
uncached,
|
||||
cache=cache,
|
||||
@ -88,6 +98,9 @@ def fetch_hltb_confidence_cached(
|
||||
),
|
||||
count_comp=count_comp,
|
||||
)
|
||||
except HLTBUnavailableError:
|
||||
# Already logged and tallied by the batch; the cache is untouched.
|
||||
return cache
|
||||
elapsed = time.monotonic() - t0
|
||||
|
||||
save_hltb_cache(cache, polls, _HLTBExtras(count_comp=count_comp))
|
||||
@ -116,9 +129,9 @@ def fetch_hltb_detail_missing(
|
||||
Also backfills ``hltb_game_id`` for any game that already has rush/leisure
|
||||
data but whose HLTB game ID was never stored (e.g. from an old cache).
|
||||
Games with both rush data and a game_id are skipped entirely. For the
|
||||
rest, temporarily removes them from the hours cache so ``fetch_hltb_times``
|
||||
will visit their detail pages. Restores prior hours for any game the
|
||||
re-fetch doesn't find.
|
||||
rest, ``fetch_hltb_times`` re-searches them and visits their detail pages
|
||||
(it never skips cached games). Prior hours win over a clean miss; a failed
|
||||
lookup writes nothing, so its old entry survives untouched.
|
||||
|
||||
Args:
|
||||
games: list of (app_id, name) tuples to check.
|
||||
@ -152,10 +165,7 @@ def fetch_hltb_detail_missing(
|
||||
hltb_game_id=game_id_cache,
|
||||
)
|
||||
|
||||
# Remove from hours cache so fetch_hltb_times will visit the detail page.
|
||||
prior_hours: dict[int, float] = {}
|
||||
for app_id, _ in missing:
|
||||
prior_hours[app_id] = cache.pop(app_id, -1.0)
|
||||
prior_hours = {aid: cache[aid] for aid, _ in missing if aid in cache}
|
||||
|
||||
n_rush = len(missing_rush)
|
||||
n_id = len(missing_id_only)
|
||||
@ -173,7 +183,11 @@ def fetch_hltb_detail_missing(
|
||||
else:
|
||||
logger.info("Backfilling HLTB game ID for %d game(s)...", n_id)
|
||||
report = reporting_progress_cb("Fetching HLTB details", len(missing), progress_cb)
|
||||
try:
|
||||
elapsed = fetch_hltb_times_timed(missing, cache, polls, report, extras)
|
||||
except HLTBUnavailableError:
|
||||
# Already logged and tallied by the batch; the cache is untouched.
|
||||
return 0
|
||||
|
||||
restore_prior_hours(cache, prior_hours)
|
||||
|
||||
@ -200,7 +214,11 @@ def get_hltb_submit_url(game_name: str) -> str | None:
|
||||
URL like ``https://howlongtobeat.com/submit/game/12345``,
|
||||
or ``None`` if the game wasn't found.
|
||||
"""
|
||||
try:
|
||||
results = fetch_hltb_times([(0, game_name)])
|
||||
except HLTBUnavailableError:
|
||||
# Already logged and tallied by the batch; the cache is untouched.
|
||||
return None
|
||||
if results and results[0].hltb_game_id:
|
||||
return f"{HLTB_BASE_URL}/submit/game/{results[0].hltb_game_id}"
|
||||
return None
|
||||
|
||||
154
steam_backlog_enforcer/_hltb_failures.py
Normal file
154
steam_backlog_enforcer/_hltb_failures.py
Normal file
@ -0,0 +1,154 @@
|
||||
"""Failure accounting for HLTB lookups.
|
||||
|
||||
A failed request (non-200, timeout, connection error, non-JSON body) is not
|
||||
evidence that a game is missing from HLTB, so the search path must keep the
|
||||
cached entry instead of writing a miss. This module tells the two apart, counts
|
||||
the failures for the end-of-run summary, and holds the per-batch circuit
|
||||
breaker that stops hammering HLTB once it starts rate-limiting us.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
# Consecutive 429s/403s after which the command sends no more requests.
|
||||
BREAKER_LIMIT = 5
|
||||
_BLOCKED = frozenset({HTTPStatus.TOO_MANY_REQUESTS, HTTPStatus.FORBIDDEN})
|
||||
SKIPPED_AFTER_RATE_LIMIT = "skipped after rate-limit"
|
||||
# A detail page that does not exist is a real answer, not a failed lookup.
|
||||
_DETAIL_NOT_A_FAILURE = frozenset({HTTPStatus.OK, HTTPStatus.NOT_FOUND})
|
||||
_PLURAL = {"timeout": "timeouts", "connection error": "connection errors"}
|
||||
|
||||
|
||||
class HLTBUnavailableError(RuntimeError):
|
||||
"""Nothing in an HLTB batch could be refreshed; the cache was left alone."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Breaker:
|
||||
"""Circuit breaker on consecutive HTTP 429 / 403 answers.
|
||||
|
||||
403 counts too: a rejected token fails every later request the same way,
|
||||
and on 2026-11-08 one let 44 doomed requests through before a manual stop.
|
||||
"""
|
||||
|
||||
consecutive_blocked: int = 0
|
||||
tripped: bool = False
|
||||
|
||||
def note(self, status: int | None) -> None:
|
||||
"""Record one request outcome (``None`` = no HTTP status at all)."""
|
||||
if status in _BLOCKED:
|
||||
self.consecutive_blocked += 1
|
||||
if self.consecutive_blocked >= BREAKER_LIMIT:
|
||||
self.tripped = True
|
||||
else:
|
||||
self.consecutive_blocked = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Tally:
|
||||
"""Failures since the last :func:`reset_failures` (one CLI command)."""
|
||||
|
||||
reasons: Counter[str] = field(default_factory=Counter)
|
||||
# App ids, so a game skipped by several passes of one command counts once.
|
||||
breaker_games: set[int] = field(default_factory=set)
|
||||
fatal: list[str] = field(default_factory=list)
|
||||
# Lookups that got a real answer (match or clean miss).
|
||||
successes: int = 0
|
||||
# Shared by every batch of the command: once HLTB rate-limits us, later
|
||||
# batches must not start hammering it again.
|
||||
breaker: Breaker = field(default_factory=Breaker)
|
||||
|
||||
|
||||
_TALLY = _Tally()
|
||||
|
||||
|
||||
def reset_failures() -> None:
|
||||
"""Start a fresh tally (call at the start of a command)."""
|
||||
_TALLY.reasons.clear()
|
||||
_TALLY.breaker_games.clear()
|
||||
_TALLY.fatal.clear()
|
||||
_TALLY.successes = 0
|
||||
_TALLY.breaker = Breaker()
|
||||
|
||||
|
||||
def shared_breaker() -> Breaker:
|
||||
"""The command-wide circuit breaker (fresh after :func:`reset_failures`)."""
|
||||
return _TALLY.breaker
|
||||
|
||||
|
||||
def record_success() -> None:
|
||||
"""Count one lookup that got a real answer from HLTB."""
|
||||
_TALLY.successes += 1
|
||||
|
||||
|
||||
def record_failure(reason: str) -> None:
|
||||
"""Count one failed lookup under *reason*."""
|
||||
_TALLY.reasons[reason] += 1
|
||||
|
||||
|
||||
def record_breaker(app_ids: Iterable[int]) -> None:
|
||||
"""Remember the games a tripped breaker left unrefreshed."""
|
||||
_TALLY.breaker_games.update(app_ids)
|
||||
|
||||
|
||||
def record_fatal(message: str) -> None:
|
||||
"""Remember a batch that refreshed nothing at all."""
|
||||
_TALLY.fatal.append(message)
|
||||
|
||||
|
||||
def had_total_failure() -> bool:
|
||||
"""True when some batch failed outright and no lookup at all succeeded."""
|
||||
return bool(_TALLY.fatal) and not _TALLY.successes
|
||||
|
||||
|
||||
def status_reason(status: int) -> str:
|
||||
"""Label for a non-200 answer."""
|
||||
return f"HTTP {status}"
|
||||
|
||||
|
||||
def exc_reason(exc: BaseException) -> str:
|
||||
"""Label for a request that raised instead of answering cleanly."""
|
||||
if isinstance(exc, TimeoutError):
|
||||
return "timeout"
|
||||
if isinstance(exc, ValueError): # json.JSONDecodeError and friends
|
||||
return "bad JSON"
|
||||
return "connection error"
|
||||
|
||||
|
||||
def detail_status_failed(status: int) -> bool:
|
||||
"""True when a detail-page answer is a failure rather than a real verdict."""
|
||||
return status not in _DETAIL_NOT_A_FAILURE
|
||||
|
||||
|
||||
def _label(reason: str) -> str:
|
||||
"""Plural form for the summary (``detail timeout`` -> ``detail timeouts``)."""
|
||||
for singular, plural in _PLURAL.items():
|
||||
if reason.endswith(singular):
|
||||
return reason[: -len(singular)] + plural
|
||||
return reason
|
||||
|
||||
|
||||
def failure_summary_lines() -> list[str]:
|
||||
"""One-line summaries of every failure since the last reset (empty if none)."""
|
||||
lines: list[str] = []
|
||||
total = sum(_TALLY.reasons.values())
|
||||
if total:
|
||||
parts = ", ".join(
|
||||
f"{_label(reason)} x{count}"
|
||||
for reason, count in _TALLY.reasons.most_common()
|
||||
)
|
||||
lines.append(
|
||||
f"HLTB: {total} lookups failed ({parts}), cache kept for those games"
|
||||
)
|
||||
if _TALLY.breaker_games:
|
||||
n_games = len(_TALLY.breaker_games)
|
||||
lines.append(f"HLTB: rate-limited, {n_games} games not refreshed")
|
||||
lines.extend(dict.fromkeys(_TALLY.fatal)) # each batch's verdict once
|
||||
return lines
|
||||
21
steam_backlog_enforcer/_hltb_pacing.py
Normal file
21
steam_backlog_enforcer/_hltb_pacing.py
Normal file
@ -0,0 +1,21 @@
|
||||
"""Pacing for HLTB requests.
|
||||
|
||||
HLTB answers 429 ("Slow your roll!") after about 60 requests in a minute:
|
||||
unpaced bursts and a 2 req/s run (2026-11-08) both stopped near 60. A full
|
||||
library refresh is ~800 games, so the single request slot
|
||||
(``MAX_CONCURRENT``) waits ``REQUEST_INTERVAL_S`` after every request before
|
||||
the next one goes out: about 55 requests per minute.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
REQUEST_INTERVAL_S = 1.1
|
||||
# Indirection so tests can drop the wait without touching asyncio itself.
|
||||
_sleep = asyncio.sleep
|
||||
|
||||
|
||||
async def pause_after_request() -> None:
|
||||
"""Hold the caller's request slot for the pacing interval."""
|
||||
await _sleep(REQUEST_INTERVAL_S)
|
||||
@ -3,147 +3,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
|
||||
from steam_backlog_enforcer._hltb_auth_refresh import apply_auth
|
||||
from steam_backlog_enforcer._hltb_detail import (
|
||||
_fetch_leisure_times,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_matching import (
|
||||
_build_result_from_best,
|
||||
_build_search_variants,
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
SKIPPED_AFTER_RATE_LIMIT,
|
||||
HLTBUnavailableError,
|
||||
record_breaker,
|
||||
record_failure,
|
||||
record_fatal,
|
||||
shared_breaker,
|
||||
status_reason,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_search_api import (
|
||||
_build_search_payload,
|
||||
_collect_candidates,
|
||||
_get_auth_info,
|
||||
_get_hltb_search_url,
|
||||
_pick_best_hltb_entry,
|
||||
forget_hltb_search_url,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
|
||||
from steam_backlog_enforcer._hltb_types import (
|
||||
_SAVE_INTERVAL,
|
||||
MAX_CONCURRENT,
|
||||
HLTBResult,
|
||||
ProgressCb,
|
||||
_AuthInfo,
|
||||
_HLTBExtras,
|
||||
save_hltb_cache,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# When extended entry has ≥ this many times more hours than the exact match,
|
||||
# prefer it even if its confidence count is lower.
|
||||
_EXTENDED_DOMINANCE_RATIO = 4.0
|
||||
# Minimum combined confidence for the dominance path (avoids picking entries
|
||||
# that have almost no data at all).
|
||||
_EXTENDED_MIN_CONFIDENCE = 3
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# HLTB API setup (done once, not per-request like the library)
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# Async fetching with shared session & progress
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SearchCtx:
|
||||
"""Shared context for HLTB search requests."""
|
||||
|
||||
session: aiohttp.ClientSession
|
||||
search_url: str
|
||||
headers: dict[str, str]
|
||||
cache: dict[int, float]
|
||||
polls: dict[int, int] = field(default_factory=dict)
|
||||
extras: _HLTBExtras = field(default_factory=_HLTBExtras)
|
||||
auth: _AuthInfo | None = None
|
||||
counter: dict[str, int] = field(default_factory=dict)
|
||||
total: int = 0
|
||||
progress_cb: ProgressCb | None = None
|
||||
def _fail_batch(message: str) -> HLTBUnavailableError:
|
||||
"""Log and remember a batch that refreshed nothing; return the error."""
|
||||
logger.warning("%s", message)
|
||||
record_fatal(message)
|
||||
return HLTBUnavailableError(message)
|
||||
|
||||
|
||||
async def _search_one(
|
||||
sem: asyncio.Semaphore,
|
||||
ctx: _SearchCtx,
|
||||
app_id: int,
|
||||
name: str,
|
||||
) -> HLTBResult | None:
|
||||
"""Search HLTB for one game via direct POST, update cache."""
|
||||
async with sem:
|
||||
result: HLTBResult | None = None
|
||||
for query_name in _build_search_variants(name):
|
||||
payload = _build_search_payload(query_name, ctx.auth)
|
||||
try:
|
||||
async with ctx.session.post(
|
||||
ctx.search_url,
|
||||
headers=ctx.headers,
|
||||
data=payload,
|
||||
) as resp:
|
||||
if resp.status != HTTPStatus.OK:
|
||||
continue
|
||||
data = await resp.json()
|
||||
candidates = _collect_candidates(query_name, data)
|
||||
# When we stripped ": subtitle" from the original name to
|
||||
# get query_name, only keep full-edition entries (those
|
||||
# whose HLTB name starts with query_name + ":"/"-") or
|
||||
# exact name/alias matches. This prevents "Vox Populi"
|
||||
# (stripped from "Vox Populi: Poland 2023") from falsely
|
||||
# matching "Vox Populi Vox Dei 2".
|
||||
if ":" in name and ":" not in query_name:
|
||||
lower_q = query_name.lower()
|
||||
candidates = [
|
||||
(e, s)
|
||||
for e, s in candidates
|
||||
if (e.get("game_name") or "").lower() == lower_q
|
||||
or (e.get("game_alias") or "").lower() == lower_q
|
||||
or (e.get("game_name") or "")
|
||||
.lower()
|
||||
.startswith(lower_q + ":")
|
||||
or (e.get("game_name") or "")
|
||||
.lower()
|
||||
.startswith(lower_q + " -")
|
||||
]
|
||||
best = _pick_best_hltb_entry(query_name, candidates)
|
||||
if best is None:
|
||||
continue
|
||||
result = _build_result_from_best(app_id, name, query_name, best)
|
||||
break
|
||||
except (TimeoutError, aiohttp.ClientError) as exc:
|
||||
logger.debug("HLTB search failed for '%s': %s", query_name, exc)
|
||||
def _skip_rate_limited(games: list[tuple[int, str]]) -> HLTBUnavailableError:
|
||||
"""Count a whole batch as not refreshed because the breaker already tripped."""
|
||||
for _game in games:
|
||||
record_failure(SKIPPED_AFTER_RATE_LIMIT)
|
||||
record_breaker(app_id for app_id, _name in games)
|
||||
msg = f"HLTB: rate-limited, skipped {len(games)} lookups"
|
||||
logger.warning("%s", msg)
|
||||
return HLTBUnavailableError(msg)
|
||||
|
||||
# Update cache immediately (miss = -1).
|
||||
if result is not None:
|
||||
ctx.cache[app_id] = result.completionist_hours
|
||||
ctx.polls[app_id] = result.comp_100_count
|
||||
ctx.extras.count_comp[app_id] = result.count_comp
|
||||
if result.hltb_game_id > 0:
|
||||
ctx.extras.hltb_game_id[app_id] = result.hltb_game_id
|
||||
ctx.counter["found"] += 1
|
||||
else:
|
||||
ctx.cache[app_id] = -1
|
||||
ctx.polls[app_id] = 0
|
||||
ctx.extras.count_comp[app_id] = 0
|
||||
|
||||
ctx.counter["done"] += 1
|
||||
done = ctx.counter["done"]
|
||||
|
||||
# Incremental save every _SAVE_INTERVAL lookups.
|
||||
if not done % _SAVE_INTERVAL:
|
||||
save_hltb_cache(ctx.cache, ctx.polls, ctx.extras)
|
||||
|
||||
# Report progress.
|
||||
if ctx.progress_cb is not None:
|
||||
ctx.progress_cb(done, ctx.total, ctx.counter["found"], name)
|
||||
|
||||
return result
|
||||
def _check_batch(ctx: _SearchCtx) -> None:
|
||||
"""Report a batch's failures; raise when not a single lookup succeeded."""
|
||||
failed = ctx.counter["failed"]
|
||||
if not failed:
|
||||
return
|
||||
logger.warning(
|
||||
"HLTB: %d/%d lookups failed in this batch; cache kept for them",
|
||||
failed,
|
||||
ctx.total,
|
||||
)
|
||||
if ctx.breaker.tripped:
|
||||
record_breaker(ctx.failed_ids)
|
||||
if failed == ctx.total:
|
||||
msg = f"HLTB: all {failed} lookups failed, nothing refreshed"
|
||||
raise _fail_batch(msg)
|
||||
|
||||
|
||||
async def _search_batch(
|
||||
@ -157,7 +86,16 @@ async def _search_batch(
|
||||
|
||||
Hours, poll counts and the extras come from the search results alone;
|
||||
:func:`_fetch_batch` layers the per-game detail pages on top of this.
|
||||
|
||||
Raises:
|
||||
HLTBUnavailableError: auth init failed, or every lookup failed. The
|
||||
caches are untouched in both cases.
|
||||
"""
|
||||
if not games:
|
||||
return []
|
||||
breaker = shared_breaker()
|
||||
if breaker.tripped:
|
||||
raise _skip_rate_limited(games)
|
||||
# 1. Discover the search URL (sync, one-time).
|
||||
search_url = _get_hltb_search_url()
|
||||
logger.info("HLTB search URL: %s", search_url)
|
||||
@ -165,12 +103,16 @@ async def _search_batch(
|
||||
timeout = aiohttp.ClientTimeout(total=20, sock_read=15)
|
||||
|
||||
# 2. Get auth info (separate session — avoids reuse issues).
|
||||
auth_failures: list[str] = []
|
||||
async with aiohttp.ClientSession(timeout=timeout) as init_session:
|
||||
auth = await _get_auth_info(search_url, init_session)
|
||||
auth = await _get_auth_info(search_url, init_session, auth_failures)
|
||||
if auth is None:
|
||||
logger.warning("Could not get HLTB auth info, aborting fetch.")
|
||||
forget_hltb_search_url()
|
||||
return []
|
||||
why = auth_failures[0] if auth_failures else "no token"
|
||||
if why == status_reason(HTTPStatus.TOO_MANY_REQUESTS):
|
||||
breaker.tripped = True
|
||||
msg = f"HLTB: rate-limited, nothing refreshed (auth init: {why})"
|
||||
raise _fail_batch(msg)
|
||||
logger.info("HLTB auth token acquired.")
|
||||
|
||||
# 3. Build shared headers for all search requests.
|
||||
@ -181,11 +123,8 @@ async def _search_batch(
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0"
|
||||
),
|
||||
"referer": "https://howlongtobeat.com/",
|
||||
"x-auth-token": auth.token,
|
||||
}
|
||||
if auth.hp_key:
|
||||
headers["x-hp-key"] = auth.hp_key
|
||||
headers["x-hp-val"] = auth.hp_val
|
||||
apply_auth(headers, auth)
|
||||
|
||||
# 4. Fire all searches through a single persistent session.
|
||||
sem = asyncio.Semaphore(MAX_CONCURRENT)
|
||||
@ -205,12 +144,14 @@ async def _search_batch(
|
||||
polls=polls,
|
||||
extras=extras,
|
||||
auth=auth,
|
||||
counter={"done": 0, "found": 0},
|
||||
counter={"done": 0, "found": 0, "failed": 0},
|
||||
total=len(games),
|
||||
progress_cb=progress_cb,
|
||||
breaker=breaker,
|
||||
)
|
||||
tasks = [_search_one(sem, ctx, app_id, name) for app_id, name in games]
|
||||
results = await asyncio.gather(*tasks)
|
||||
_check_batch(ctx)
|
||||
|
||||
return [r for r in results if r is not None]
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@ from typing import Any
|
||||
import aiohttp
|
||||
from howlongtobeatpy.HTMLRequests import HTMLRequests
|
||||
|
||||
from steam_backlog_enforcer._hltb_failures import exc_reason, status_reason
|
||||
from steam_backlog_enforcer._hltb_matching import (
|
||||
_find_best_extended,
|
||||
_find_exact_match,
|
||||
@ -91,14 +92,20 @@ def forget_hltb_search_url() -> None:
|
||||
async def _get_auth_info(
|
||||
search_url: str,
|
||||
session: aiohttp.ClientSession,
|
||||
failures: list[str] | None = None,
|
||||
) -> _AuthInfo | None:
|
||||
"""Fetch the HLTB auth token and honeypot key/val (one GET request)."""
|
||||
"""Fetch the HLTB auth token and honeypot key/val (one GET request).
|
||||
|
||||
On failure returns ``None`` and, when *failures* is given, appends why
|
||||
(``HTTP 429``, ``timeout``, ...) so the caller can report it.
|
||||
"""
|
||||
init_url = search_url + "/init"
|
||||
ts = int(time.time() * 1000)
|
||||
headers = {
|
||||
"User-Agent": _USER_AGENT,
|
||||
"referer": "https://howlongtobeat.com/",
|
||||
}
|
||||
reason = "no token"
|
||||
try:
|
||||
async with session.get(
|
||||
init_url,
|
||||
@ -108,15 +115,19 @@ async def _get_auth_info(
|
||||
if resp.status == HTTPStatus.OK:
|
||||
data = await resp.json()
|
||||
token: str | None = data.get("token")
|
||||
if token is None:
|
||||
return None
|
||||
if token is not None:
|
||||
return _AuthInfo(
|
||||
token=token,
|
||||
hp_key=data.get("hpKey", ""),
|
||||
hp_val=data.get("hpVal", ""),
|
||||
)
|
||||
except TimeoutError, aiohttp.ClientError:
|
||||
logger.warning("Failed to get HLTB auth token")
|
||||
else:
|
||||
reason = status_reason(resp.status)
|
||||
except (TimeoutError, aiohttp.ClientError, ValueError) as exc:
|
||||
reason = exc_reason(exc)
|
||||
logger.warning("Failed to get HLTB auth token (%s)", reason)
|
||||
if failures is not None:
|
||||
failures.append(reason)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
198
steam_backlog_enforcer/_hltb_search_one.py
Normal file
198
steam_backlog_enforcer/_hltb_search_one.py
Normal file
@ -0,0 +1,198 @@
|
||||
"""One HLTB game lookup: query each name variant, classify, write the cache.
|
||||
|
||||
Split out of :mod:`steam_backlog_enforcer._hltb_search` to keep both files
|
||||
under the 250-line cap. A failed request never writes a miss: only a lookup
|
||||
whose every variant answered a clean 200 with nothing usable does.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
|
||||
from steam_backlog_enforcer._hltb_auth_refresh import ensure_fresh_auth
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
SKIPPED_AFTER_RATE_LIMIT,
|
||||
Breaker,
|
||||
exc_reason,
|
||||
record_failure,
|
||||
record_success,
|
||||
status_reason,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_matching import (
|
||||
_build_result_from_best,
|
||||
_build_search_variants,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_pacing import pause_after_request
|
||||
from steam_backlog_enforcer._hltb_search_api import (
|
||||
_build_search_payload,
|
||||
_collect_candidates,
|
||||
_pick_best_hltb_entry,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_types import (
|
||||
_SAVE_INTERVAL,
|
||||
HLTBResult,
|
||||
ProgressCb,
|
||||
_AuthInfo,
|
||||
_HLTBExtras,
|
||||
save_hltb_cache,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SearchCtx:
|
||||
"""Shared context for HLTB search requests."""
|
||||
|
||||
session: aiohttp.ClientSession
|
||||
search_url: str
|
||||
headers: dict[str, str]
|
||||
cache: dict[int, float]
|
||||
polls: dict[int, int] = field(default_factory=dict)
|
||||
extras: _HLTBExtras = field(default_factory=_HLTBExtras)
|
||||
auth: _AuthInfo | None = None
|
||||
counter: dict[str, int] = field(default_factory=dict)
|
||||
total: int = 0
|
||||
progress_cb: ProgressCb | None = None
|
||||
breaker: Breaker = field(default_factory=Breaker)
|
||||
# When ``auth`` was issued (monotonic) and the lock for renewing it.
|
||||
auth_at: float = field(default_factory=time.monotonic)
|
||||
auth_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||
# Games whose lookup failed (the breaker reports them by id).
|
||||
failed_ids: set[int] = field(default_factory=set)
|
||||
# Only flush incrementally once something was actually written.
|
||||
dirty: bool = False
|
||||
|
||||
|
||||
_Best = tuple[dict[str, Any], float]
|
||||
|
||||
|
||||
def _full_edition_candidates(
|
||||
name: str, query_name: str, candidates: list[_Best]
|
||||
) -> list[_Best]:
|
||||
"""Drop unrelated entries when ``query_name`` is ``name`` minus a subtitle.
|
||||
|
||||
When we stripped ": subtitle" from the original name to get query_name,
|
||||
only keep full-edition entries (those whose HLTB name starts with
|
||||
query_name + ":"/"-") or exact name/alias matches. This prevents "Vox
|
||||
Populi" (stripped from "Vox Populi: Poland 2023") from falsely matching
|
||||
"Vox Populi Vox Dei 2".
|
||||
"""
|
||||
if ":" not in name or ":" in query_name:
|
||||
return candidates
|
||||
lower_q = query_name.lower()
|
||||
kept: list[_Best] = []
|
||||
for entry, sim in candidates:
|
||||
game_name = (entry.get("game_name") or "").lower()
|
||||
alias = (entry.get("game_alias") or "").lower()
|
||||
if lower_q in {game_name, alias} or game_name.startswith(
|
||||
(lower_q + ":", lower_q + " -")
|
||||
):
|
||||
kept.append((entry, sim))
|
||||
return kept
|
||||
|
||||
|
||||
async def _query_variant(
|
||||
ctx: _SearchCtx, name: str, query_name: str
|
||||
) -> tuple[_Best | None, str | None]:
|
||||
"""Run one search variant.
|
||||
|
||||
Returns ``(best, None)`` on a match, ``(None, None)`` on a clean 200 with
|
||||
no usable entry, and ``(None, reason)`` when the request itself failed.
|
||||
"""
|
||||
payload = _build_search_payload(query_name, ctx.auth)
|
||||
try:
|
||||
async with ctx.session.post(
|
||||
ctx.search_url, headers=ctx.headers, data=payload
|
||||
) as resp:
|
||||
ctx.breaker.note(resp.status)
|
||||
if resp.status != HTTPStatus.OK:
|
||||
return None, status_reason(resp.status)
|
||||
data = await resp.json()
|
||||
except (TimeoutError, aiohttp.ClientError, ValueError) as exc:
|
||||
ctx.breaker.note(None)
|
||||
return None, exc_reason(exc)
|
||||
finally:
|
||||
await pause_after_request()
|
||||
if not isinstance(data, dict):
|
||||
return None, "bad JSON"
|
||||
candidates = _full_edition_candidates(
|
||||
name, query_name, _collect_candidates(query_name, data)
|
||||
)
|
||||
return _pick_best_hltb_entry(query_name, candidates), None
|
||||
|
||||
|
||||
def _record_outcome(
|
||||
ctx: _SearchCtx, app_id: int, result: HLTBResult | None, failure: str | None
|
||||
) -> None:
|
||||
"""Write a match or a clean miss; a failed lookup keeps the old entry."""
|
||||
if result is not None:
|
||||
ctx.cache[app_id] = result.completionist_hours
|
||||
ctx.polls[app_id] = result.comp_100_count
|
||||
ctx.extras.count_comp[app_id] = result.count_comp
|
||||
if result.hltb_game_id > 0:
|
||||
ctx.extras.hltb_game_id[app_id] = result.hltb_game_id
|
||||
ctx.counter["found"] += 1
|
||||
ctx.dirty = True
|
||||
record_success()
|
||||
elif failure is not None:
|
||||
ctx.counter["failed"] += 1
|
||||
ctx.failed_ids.add(app_id)
|
||||
record_failure(failure)
|
||||
else:
|
||||
# Every variant answered cleanly with nothing usable: a real miss.
|
||||
ctx.cache[app_id] = -1
|
||||
ctx.polls[app_id] = 0
|
||||
ctx.extras.count_comp[app_id] = 0
|
||||
ctx.dirty = True
|
||||
record_success()
|
||||
|
||||
|
||||
async def _search_one(
|
||||
sem: asyncio.Semaphore,
|
||||
ctx: _SearchCtx,
|
||||
app_id: int,
|
||||
name: str,
|
||||
) -> HLTBResult | None:
|
||||
"""Search HLTB for one game via direct POST, update cache."""
|
||||
async with sem:
|
||||
result: HLTBResult | None = None
|
||||
failure: str | None = None
|
||||
for query_name in _build_search_variants(name):
|
||||
if ctx.breaker.tripped:
|
||||
failure = failure or SKIPPED_AFTER_RATE_LIMIT
|
||||
break
|
||||
auth_failure = await ensure_fresh_auth(ctx)
|
||||
if auth_failure is not None:
|
||||
failure = failure or auth_failure
|
||||
break
|
||||
best, reason = await _query_variant(ctx, name, query_name)
|
||||
if best is not None:
|
||||
result = _build_result_from_best(app_id, name, query_name, best)
|
||||
break
|
||||
failure = failure or reason
|
||||
if result is None and failure not in {None, SKIPPED_AFTER_RATE_LIMIT}:
|
||||
logger.warning(
|
||||
"HLTB lookup failed for '%s' (%s), cache kept", name, failure
|
||||
)
|
||||
_record_outcome(ctx, app_id, result, failure)
|
||||
|
||||
ctx.counter["done"] += 1
|
||||
done = ctx.counter["done"]
|
||||
|
||||
# Incremental save every _SAVE_INTERVAL lookups.
|
||||
if ctx.dirty and not done % _SAVE_INTERVAL:
|
||||
save_hltb_cache(ctx.cache, ctx.polls, ctx.extras)
|
||||
|
||||
# Report progress.
|
||||
if ctx.progress_cb is not None:
|
||||
ctx.progress_cb(done, ctx.total, ctx.counter["found"], name)
|
||||
|
||||
return result
|
||||
@ -13,7 +13,8 @@ from steam_backlog_enforcer.config import CONFIG_DIR, _atomic_write
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HLTB_CACHE_FILE = CONFIG_DIR / "hltb_cache.json"
|
||||
MAX_CONCURRENT = 60 # parallel requests to HLTB
|
||||
# Requests in flight to HLTB; each slot is paced by _hltb_pacing.
|
||||
MAX_CONCURRENT = 1
|
||||
_SAVE_INTERVAL = 50 # flush cache to disk every N results
|
||||
MIN_SIMILARITY = 0.5
|
||||
HLTB_BASE_URL = "https://howlongtobeat.com"
|
||||
|
||||
@ -5,14 +5,10 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from steam_backlog_enforcer._hltb_confidence import fetch_hltb_confidence_cached
|
||||
from steam_backlog_enforcer._hltb_confidence import refresh_confidence
|
||||
from steam_backlog_enforcer._hltb_types import (
|
||||
_HLTBExtras,
|
||||
load_hltb_cache,
|
||||
load_hltb_count_comp_cache,
|
||||
load_hltb_polls_cache,
|
||||
restore_prior_hours,
|
||||
save_hltb_cache,
|
||||
)
|
||||
from steam_backlog_enforcer._polls_reporting import (
|
||||
_backfill_polls_for_finished,
|
||||
@ -105,27 +101,8 @@ def _refresh_candidate_confidence_batch(
|
||||
else:
|
||||
_echo(f" Refreshing HLTB confidence for {len(refresh_slice)} candidate(s)...")
|
||||
|
||||
cache = load_hltb_cache()
|
||||
polls = load_hltb_polls_cache()
|
||||
count_comp = load_hltb_count_comp_cache()
|
||||
app_ids = [game.app_id for game in refresh_slice]
|
||||
names = [(game.app_id, game.name) for game in refresh_slice]
|
||||
prior_hours = {aid: cache.get(aid, -1) for aid in app_ids}
|
||||
|
||||
for aid in app_ids:
|
||||
cache.pop(aid, None)
|
||||
polls.pop(aid, None)
|
||||
count_comp.pop(aid, None)
|
||||
save_hltb_cache(cache, polls, _HLTBExtras(count_comp=count_comp))
|
||||
|
||||
fetch_hltb_confidence_cached(names)
|
||||
|
||||
refreshed_hours = load_hltb_cache()
|
||||
refreshed_polls = load_hltb_polls_cache()
|
||||
refreshed_count_comp = load_hltb_count_comp_cache()
|
||||
restore_prior_hours(refreshed_hours, prior_hours)
|
||||
save_hltb_cache(
|
||||
refreshed_hours, refreshed_polls, _HLTBExtras(count_comp=refreshed_count_comp)
|
||||
_, refreshed_polls, refreshed_count_comp = refresh_confidence(
|
||||
[(game.app_id, game.name) for game in refresh_slice]
|
||||
)
|
||||
|
||||
for game in refresh_slice:
|
||||
|
||||
@ -3,8 +3,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
failure_summary_lines,
|
||||
had_total_failure,
|
||||
reset_failures,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_types import (
|
||||
_read_raw_cache,
|
||||
)
|
||||
@ -131,7 +137,22 @@ def cmd_stats(_config: Config, state: State) -> None:
|
||||
3. Leisure — comp_100_h (slowest 100 %) + DLC leisure per HLTB.
|
||||
4. Worst — absolute maximum recorded time (any category) per HLTB.
|
||||
5. Your play style — extrapolated from completed-game calibration vs HLTB.
|
||||
|
||||
Ends with one line per kind of HLTB failure (the cache is kept for those
|
||||
games) and exits 1 when some HLTB batch refreshed nothing at all.
|
||||
"""
|
||||
reset_failures()
|
||||
try:
|
||||
_print_stats(_config, state)
|
||||
finally:
|
||||
for line in failure_summary_lines():
|
||||
_echo(line)
|
||||
if had_total_failure():
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _print_stats(_config: Config, state: State) -> None:
|
||||
"""Gather the stats data and print every scenario for :func:`cmd_stats`."""
|
||||
games = require_snapshot_games()
|
||||
if games is None:
|
||||
return
|
||||
|
||||
93
steam_backlog_enforcer/tests/_hltb_fake_transport.py
Normal file
93
steam_backlog_enforcer/tests/_hltb_fake_transport.py
Normal file
@ -0,0 +1,93 @@
|
||||
"""Fake aiohttp transport for HLTB search tests: no request leaves the process."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Self
|
||||
|
||||
MATCH = {
|
||||
"data": [
|
||||
{
|
||||
"game_name": "Vox Populi",
|
||||
"game_alias": "",
|
||||
"game_type": "game",
|
||||
"game_id": 77,
|
||||
"comp_100": 36000,
|
||||
"comp_100_count": 42,
|
||||
"count_comp": 300,
|
||||
}
|
||||
]
|
||||
}
|
||||
NO_MATCH: dict[str, Any] = {"data": []}
|
||||
|
||||
|
||||
class FakeResp:
|
||||
"""Minimal aiohttp response stand-in."""
|
||||
|
||||
def __init__(self, status: int, body: dict[str, Any] | None = None) -> None:
|
||||
"""Store the canned status and JSON body (``None`` = not JSON)."""
|
||||
self.status = status
|
||||
self._body = body
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
"""Enter."""
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
"""Exit."""
|
||||
|
||||
async def json(self) -> dict[str, Any]:
|
||||
"""Return the body, or fail like a garbage payload would."""
|
||||
if self._body is None:
|
||||
msg = "Expecting value"
|
||||
raise json.JSONDecodeError(msg, "<html>", 0)
|
||||
return self._body
|
||||
|
||||
|
||||
class FakeSession:
|
||||
"""Fake session routing each search POST by its first search term."""
|
||||
|
||||
def __init__(self, routes: dict[str, object]) -> None:
|
||||
"""Map a query's first word to a _Resp or an exception to raise."""
|
||||
self.routes = routes
|
||||
self.posts: list[str] = []
|
||||
|
||||
def post(self, _url: str, *, headers: object, data: str) -> FakeResp:
|
||||
"""Return (or raise) the canned outcome for this query."""
|
||||
del headers
|
||||
terms = json.loads(data)["searchTerms"]
|
||||
query = " ".join(terms)
|
||||
self.posts.append(query)
|
||||
outcome = self.routes.get(query, self.routes.get("*"))
|
||||
if isinstance(outcome, BaseException):
|
||||
raise outcome
|
||||
assert isinstance(outcome, FakeResp)
|
||||
return outcome
|
||||
|
||||
|
||||
class FakeInitSession:
|
||||
"""Fake ClientSession: GET /init answers ``status``; POSTs use ``routes``."""
|
||||
|
||||
def __init__(self, status: int | BaseException, routes: dict[str, object]) -> None:
|
||||
"""Store the canned init outcome and search routes."""
|
||||
self.status = status
|
||||
self.search = FakeSession(routes)
|
||||
self.gets = 0
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
"""Enter."""
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
"""Exit."""
|
||||
|
||||
def get(self, *_a: object, **_k: object) -> FakeResp:
|
||||
"""Answer the auth init."""
|
||||
self.gets += 1
|
||||
if isinstance(self.status, BaseException):
|
||||
raise self.status
|
||||
return FakeResp(self.status, {"token": "tok"})
|
||||
|
||||
def post(self, url: str, *, headers: object, data: str) -> FakeResp:
|
||||
"""Delegate searches."""
|
||||
return self.search.post(url, headers=headers, data=data)
|
||||
@ -16,6 +16,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer._hltb_failures import reset_failures
|
||||
from steam_backlog_enforcer._hltb_search_api import forget_hltb_search_url
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@ -65,16 +66,8 @@ def _isolate_cover_art(tmp_path: Path) -> Iterator[None]:
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _forget_hltb_search_url() -> Iterator[None]:
|
||||
"""Start every test without a remembered HLTB search endpoint.
|
||||
|
||||
``_get_hltb_search_url`` memoises a discovered URL for half an hour; a
|
||||
value left behind by one test would hide the discovery path in the next.
|
||||
"""
|
||||
forget_hltb_search_url()
|
||||
yield
|
||||
forget_hltb_search_url()
|
||||
async def _no_pause(_seconds: float) -> None:
|
||||
"""Stand-in for the HLTB pacing sleep."""
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@ -83,7 +76,13 @@ def _forget_hltb_search_url() -> Iterator[None]:
|
||||
|
||||
``_get_hltb_search_url`` memoises a discovered URL for half an hour; a
|
||||
value left behind by one test would hide the discovery path in the next.
|
||||
The HLTB failure tally and its shared circuit breaker are reset too: a
|
||||
breaker tripped by one test would make the next one skip every lookup.
|
||||
The per-request HLTB pacing pause is dropped so tests never wait on it.
|
||||
"""
|
||||
forget_hltb_search_url()
|
||||
reset_failures()
|
||||
with patch("steam_backlog_enforcer._hltb_pacing._sleep", _no_pause):
|
||||
yield
|
||||
forget_hltb_search_url()
|
||||
reset_failures()
|
||||
|
||||
@ -10,9 +10,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import aiohttp
|
||||
|
||||
from steam_backlog_enforcer._hltb_matching import _similarity
|
||||
from steam_backlog_enforcer._hltb_search import _AuthInfo
|
||||
from steam_backlog_enforcer._hltb_search_api import _get_auth_info, _get_hltb_search_url
|
||||
from steam_backlog_enforcer._hltb_types import (
|
||||
_AuthInfo,
|
||||
load_hltb_cache,
|
||||
save_hltb_cache,
|
||||
)
|
||||
|
||||
194
steam_backlog_enforcer/tests/test_hltb_auth_refresh.py
Normal file
194
steam_backlog_enforcer/tests/test_hltb_auth_refresh.py
Normal file
@ -0,0 +1,194 @@
|
||||
"""A long paced batch renews the HLTB token before it expires.
|
||||
|
||||
Seen 2026-11-08: at ~55 requests/min every search ~3 min after /init came
|
||||
back 403 while a fresh token got 200 for the same game. Fake transport and a
|
||||
fake clock only: no request leaves the process and nothing sleeps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Self, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
from steam_backlog_enforcer import _hltb_auth_refresh
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
BREAKER_LIMIT,
|
||||
SKIPPED_AFTER_RATE_LIMIT,
|
||||
Breaker,
|
||||
failure_summary_lines,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
|
||||
from steam_backlog_enforcer._hltb_types import _AuthInfo
|
||||
from steam_backlog_enforcer.tests._hltb_fake_transport import MATCH, FakeResp
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import aiohttp
|
||||
|
||||
_AGE = _hltb_auth_refresh.AUTH_MAX_AGE_S
|
||||
|
||||
|
||||
class _RotatingSession:
|
||||
"""Fake session: each /init GET hands out a new token; POSTs record auth."""
|
||||
|
||||
def __init__(self, init_status: int = 200, post_status: int = 200) -> None:
|
||||
self.init_status = init_status
|
||||
self.post_status = post_status
|
||||
self.gets = 0
|
||||
# (x-auth-token header, honeypot value in the body) per POST.
|
||||
self.posts: list[tuple[str | None, str | None]] = []
|
||||
|
||||
def get(self, *_a: object, **_k: object) -> FakeResp:
|
||||
self.gets += 1
|
||||
n = self.gets
|
||||
return FakeResp(
|
||||
self.init_status, {"token": f"tok{n}", "hpKey": "hp", "hpVal": f"v{n}"}
|
||||
)
|
||||
|
||||
def post(self, _url: str, *, headers: dict[str, str], data: str) -> FakeResp:
|
||||
self.posts.append((headers.get("x-auth-token"), json.loads(data).get("hp")))
|
||||
body = MATCH if self.post_status == 200 else None
|
||||
return FakeResp(self.post_status, body)
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _ListBodySession(_RotatingSession):
|
||||
"""Fake session whose searches answer 200 with a JSON list, not an object."""
|
||||
|
||||
def post(self, _url: str, *, headers: dict[str, str], data: str) -> FakeResp:
|
||||
del headers, data
|
||||
return FakeResp(200, cast("dict[str, object]", []))
|
||||
|
||||
|
||||
class _Clock:
|
||||
"""Fake monotonic clock advanced by the test."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.t = 1000.0
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.t
|
||||
|
||||
|
||||
def _ctx(session: _RotatingSession, clock: _Clock) -> _SearchCtx:
|
||||
return _SearchCtx(
|
||||
session=cast("aiohttp.ClientSession", session),
|
||||
search_url="https://example.invalid/api/finder",
|
||||
headers={"x-auth-token": "tok0", "x-hp-key": "hp", "x-hp-val": "v0"},
|
||||
cache={},
|
||||
auth=_AuthInfo("tok0", "hp", "v0"),
|
||||
counter={"done": 0, "found": 0, "failed": 0},
|
||||
breaker=Breaker(),
|
||||
auth_at=clock.t,
|
||||
)
|
||||
|
||||
|
||||
def _lookups(ctx: _SearchCtx, clock: _Clock, steps: list[float]) -> None:
|
||||
"""Run one lookup per step, advancing the clock by that many seconds first."""
|
||||
|
||||
async def run() -> None:
|
||||
sem = asyncio.Semaphore(1)
|
||||
for i, step in enumerate(steps):
|
||||
clock.t += step
|
||||
await _search_one(sem, ctx, i + 1, "Vox Populi")
|
||||
|
||||
with patch.object(_hltb_auth_refresh, "_now", clock):
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_young_token_is_not_refreshed() -> None:
|
||||
"""Within the max age no extra /init request is made."""
|
||||
session, clock = _RotatingSession(), _Clock()
|
||||
_lookups(_ctx(session, clock), clock, [0, _AGE / 2])
|
||||
assert session.gets == 0
|
||||
assert session.posts == [("tok0", "v0"), ("tok0", "v0")]
|
||||
|
||||
|
||||
def test_aged_token_is_refreshed_into_headers_and_body() -> None:
|
||||
"""Past the max age the next POST carries the new token and honeypot."""
|
||||
session, clock = _RotatingSession(), _Clock()
|
||||
ctx = _ctx(session, clock)
|
||||
_lookups(ctx, clock, [0, _AGE + 1])
|
||||
assert session.gets == 1
|
||||
assert session.posts == [("tok0", "v0"), ("tok1", "v1")]
|
||||
assert ctx.auth == _AuthInfo("tok1", "hp", "v1")
|
||||
|
||||
|
||||
def test_one_refresh_per_elapsed_interval() -> None:
|
||||
"""Many lookups inside one interval share a single refresh."""
|
||||
session, clock = _RotatingSession(), _Clock()
|
||||
_lookups(_ctx(session, clock), clock, [_AGE + 1, 1, 1, 1, _AGE + 1, 1])
|
||||
assert session.gets == 2
|
||||
assert [tok for tok, _hp in session.posts] == [
|
||||
"tok1",
|
||||
"tok1",
|
||||
"tok1",
|
||||
"tok1",
|
||||
"tok2",
|
||||
"tok2",
|
||||
]
|
||||
|
||||
|
||||
def test_failed_refresh_stops_sending_and_trips_the_breaker() -> None:
|
||||
"""A refresh that gets 429 never falls back to the stale token."""
|
||||
session, clock = _RotatingSession(init_status=429), _Clock()
|
||||
ctx = _ctx(session, clock)
|
||||
_lookups(ctx, clock, [_AGE + 1, 1, 1])
|
||||
assert session.gets == 1
|
||||
assert session.posts == []
|
||||
assert ctx.breaker.tripped
|
||||
assert ctx.counter["failed"] == 3
|
||||
summary = failure_summary_lines()[0]
|
||||
assert "auth refresh HTTP 429 x1" in summary
|
||||
assert f"{SKIPPED_AFTER_RATE_LIMIT} x2" in summary
|
||||
|
||||
|
||||
def test_consecutive_403s_trip_the_breaker() -> None:
|
||||
"""A rejected token cannot turn into hundreds of doomed requests."""
|
||||
session, clock = _RotatingSession(post_status=403), _Clock()
|
||||
ctx = _ctx(session, clock)
|
||||
_lookups(ctx, clock, [0] * (BREAKER_LIMIT + 3))
|
||||
assert len(session.posts) == BREAKER_LIMIT
|
||||
assert ctx.breaker.tripped
|
||||
assert ctx.counter["failed"] == BREAKER_LIMIT + 3
|
||||
|
||||
|
||||
def test_concurrent_lookups_share_one_refresh() -> None:
|
||||
"""A lookup that waited on the lock reuses the token renewed meanwhile."""
|
||||
session, clock = _RotatingSession(), _Clock()
|
||||
ctx = _ctx(session, clock)
|
||||
clock.t += _AGE + 1
|
||||
|
||||
async def yielding_pause(_seconds: float) -> None:
|
||||
await asyncio.sleep(0) # let the other lookup reach the lock
|
||||
|
||||
async def run() -> list[str | None]:
|
||||
return list(
|
||||
await asyncio.gather(
|
||||
_hltb_auth_refresh.ensure_fresh_auth(ctx),
|
||||
_hltb_auth_refresh.ensure_fresh_auth(ctx),
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(_hltb_auth_refresh, "_now", clock),
|
||||
patch("steam_backlog_enforcer._hltb_pacing._sleep", yielding_pause),
|
||||
):
|
||||
assert asyncio.run(run()) == [None, None]
|
||||
assert session.gets == 1
|
||||
|
||||
|
||||
def test_non_object_json_is_a_failure() -> None:
|
||||
"""A 200 whose body is not a JSON object is "bad JSON", never a miss."""
|
||||
session, clock = _ListBodySession(), _Clock()
|
||||
ctx = _ctx(session, clock)
|
||||
ctx.cache[1] = 50.0
|
||||
_lookups(ctx, clock, [0])
|
||||
assert ctx.cache[1] == 50.0
|
||||
assert "bad JSON x1" in failure_summary_lines()[0]
|
||||
@ -4,8 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from steam_backlog_enforcer._hltb_search import _AuthInfo
|
||||
from steam_backlog_enforcer._hltb_search_api import _build_search_payload
|
||||
from steam_backlog_enforcer._hltb_types import _AuthInfo
|
||||
|
||||
|
||||
class TestBuildSearchPayload:
|
||||
|
||||
@ -0,0 +1,93 @@
|
||||
"""A failed HLTB batch leaves every cached entry point's cache alone.
|
||||
|
||||
No network: the batch functions are faked at the module boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from steam_backlog_enforcer import _hltb_cached, _hltb_confidence
|
||||
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
|
||||
|
||||
_CONF = "steam_backlog_enforcer._hltb_confidence"
|
||||
_DOWN = HLTBUnavailableError("HLTB: rate-limited, nothing refreshed")
|
||||
|
||||
|
||||
def _found(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Fake search that finds every game."""
|
||||
for aid, _name in games:
|
||||
cache[aid], polls[aid], count_comp[aid] = 5.0, 7, 40
|
||||
return []
|
||||
|
||||
|
||||
def test_cached_fetch_saves_new_results() -> None:
|
||||
"""Uncached games are searched and the result is saved."""
|
||||
with (
|
||||
patch(f"{_CONF}.fetch_hltb_confidence", side_effect=_found),
|
||||
patch(f"{_CONF}.save_hltb_cache") as save,
|
||||
):
|
||||
result = _hltb_confidence.fetch_hltb_confidence_cached([(1, "Celeste")])
|
||||
assert result == {1: 5.0}
|
||||
save.assert_called_once()
|
||||
|
||||
|
||||
def test_cached_fetch_keeps_cache_when_hltb_is_down() -> None:
|
||||
"""An unavailable batch returns the cache as loaded and saves nothing."""
|
||||
with (
|
||||
patch(f"{_CONF}.load_hltb_cache", return_value={2: 9.0}),
|
||||
patch(f"{_CONF}.fetch_hltb_confidence", side_effect=_DOWN),
|
||||
patch(f"{_CONF}.save_hltb_cache") as save,
|
||||
):
|
||||
result = _hltb_confidence.fetch_hltb_confidence_cached([(1, "Celeste")])
|
||||
assert result == {2: 9.0}
|
||||
save.assert_not_called()
|
||||
|
||||
|
||||
def test_detail_missing_reports_nothing_fetched_when_hltb_is_down() -> None:
|
||||
"""The detail backfill returns 0 and writes nothing when HLTB is down."""
|
||||
with (
|
||||
patch(f"{_CONF}.fetch_hltb_times_timed", side_effect=_DOWN),
|
||||
patch(f"{_CONF}.save_hltb_cache") as save,
|
||||
):
|
||||
assert _hltb_confidence.fetch_hltb_detail_missing([(1, "Celeste")]) == 0
|
||||
save.assert_not_called()
|
||||
|
||||
|
||||
def test_submit_url_is_none_when_hltb_is_down() -> None:
|
||||
"""No URL instead of a crash when the lookup cannot run."""
|
||||
with patch(f"{_CONF}.fetch_hltb_times", side_effect=_DOWN):
|
||||
assert _hltb_confidence.get_hltb_submit_url("Celeste") is None
|
||||
|
||||
|
||||
def test_refresh_keeps_cache_when_hltb_is_down() -> None:
|
||||
"""A re-search that cannot run returns the loaded caches and saves nothing."""
|
||||
with (
|
||||
patch(f"{_CONF}.load_hltb_cache", return_value={1: 9.0}),
|
||||
patch(f"{_CONF}.load_hltb_polls_cache", return_value={1: 30}),
|
||||
patch(f"{_CONF}.load_hltb_count_comp_cache", return_value={1: 80}),
|
||||
patch(f"{_CONF}.fetch_hltb_confidence", side_effect=_DOWN),
|
||||
patch(f"{_CONF}.save_hltb_cache") as save,
|
||||
):
|
||||
result = _hltb_confidence.refresh_confidence([(1, "Celeste")])
|
||||
assert result == ({1: 9.0}, {1: 30}, {1: 80})
|
||||
save.assert_not_called()
|
||||
|
||||
|
||||
def test_times_cached_keeps_cache_when_hltb_is_down() -> None:
|
||||
"""The times fetch returns the cache as loaded and saves nothing."""
|
||||
cached = "steam_backlog_enforcer._hltb_cached"
|
||||
with (
|
||||
patch(f"{cached}.load_hltb_cache", return_value={2: 9.0}),
|
||||
patch(f"{cached}.fetch_hltb_times_timed", side_effect=_DOWN),
|
||||
patch(f"{cached}.save_hltb_cache") as save,
|
||||
):
|
||||
result = _hltb_cached.fetch_hltb_times_cached([(1, "Celeste")])
|
||||
assert result == {2: 9.0}
|
||||
save.assert_not_called()
|
||||
@ -6,8 +6,11 @@ import asyncio
|
||||
from typing import Self
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer._hltb_confidence import fetch_hltb_confidence_cached
|
||||
from steam_backlog_enforcer._hltb_search import _AuthInfo
|
||||
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
|
||||
from steam_backlog_enforcer._hltb_types import _AuthInfo
|
||||
from steam_backlog_enforcer.hltb import (
|
||||
_fetch_batch_confidence_only,
|
||||
fetch_hltb_confidence,
|
||||
@ -39,11 +42,9 @@ class TestConfidenceHelpers:
|
||||
patch(f"{PKG}.aiohttp.TCPConnector"),
|
||||
patch(f"{PKG}._get_hltb_search_url", return_value="https://example"),
|
||||
patch(f"{PKG}._get_auth_info", return_value=None),
|
||||
pytest.raises(HLTBUnavailableError, match="nothing refreshed"),
|
||||
):
|
||||
result = asyncio.run(
|
||||
_fetch_batch_confidence_only([(1, "Game")], {}, {}, None),
|
||||
)
|
||||
assert result == []
|
||||
asyncio.run(_fetch_batch_confidence_only([(1, "Game")], {}, {}, None))
|
||||
|
||||
def test_fetch_batch_confidence_only_handles_empty_hp_and_default_counts(
|
||||
self,
|
||||
|
||||
131
steam_backlog_enforcer/tests/test_hltb_failure_reporting.py
Normal file
131
steam_backlog_enforcer/tests/test_hltb_failure_reporting.py
Normal file
@ -0,0 +1,131 @@
|
||||
"""Detail-page failures are counted; ``stats`` prints the tally and fails loud."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Self
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer import _stats
|
||||
from steam_backlog_enforcer._hltb_comp_extract import _fetch_detail_one
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
failure_summary_lines,
|
||||
record_failure,
|
||||
record_fatal,
|
||||
record_success,
|
||||
reset_failures,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_tally() -> None:
|
||||
reset_failures()
|
||||
|
||||
|
||||
class _Page:
|
||||
"""Fake detail-page response."""
|
||||
|
||||
def __init__(self, status: int) -> None:
|
||||
"""Store the status."""
|
||||
self.status = status
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
"""Enter."""
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
"""Exit."""
|
||||
|
||||
async def text(self) -> str:
|
||||
"""Empty page."""
|
||||
return ""
|
||||
|
||||
|
||||
def _detail(outcome: object) -> None:
|
||||
session = MagicMock()
|
||||
if isinstance(outcome, BaseException):
|
||||
session.get.side_effect = outcome
|
||||
else:
|
||||
session.get.return_value = outcome
|
||||
asyncio.run(_fetch_detail_one(asyncio.Semaphore(1), session, 77))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("outcome", "label"),
|
||||
[
|
||||
(_Page(429), "detail HTTP 429"),
|
||||
(_Page(500), "detail HTTP 500"),
|
||||
(TimeoutError(), "detail timeouts"),
|
||||
(aiohttp.ClientConnectionError(), "detail connection errors"),
|
||||
],
|
||||
)
|
||||
def test_detail_failure_is_counted(outcome: object, label: str) -> None:
|
||||
"""A failed detail fetch shows up in the summary line."""
|
||||
_detail(outcome)
|
||||
assert f"{label} x1" in failure_summary_lines()[0]
|
||||
|
||||
|
||||
def test_missing_detail_page_is_not_a_failure() -> None:
|
||||
"""A 404 is HLTB's real answer, not a failed lookup."""
|
||||
_detail(_Page(404))
|
||||
assert failure_summary_lines() == []
|
||||
|
||||
|
||||
def _stats_with(effect: object) -> None:
|
||||
with patch.object(_stats, "_print_stats", side_effect=effect):
|
||||
_stats.cmd_stats(MagicMock(), MagicMock())
|
||||
|
||||
|
||||
def test_stats_ends_with_failure_line_and_exits_zero(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""Old data survived: report the failures, exit 0."""
|
||||
_stats_with(lambda *_a: record_failure("HTTP 429"))
|
||||
out = capsys.readouterr().out.rstrip().splitlines()
|
||||
assert out[-1] == (
|
||||
"HLTB: 1 lookups failed (HTTP 429 x1), cache kept for those games"
|
||||
)
|
||||
|
||||
|
||||
def test_stats_exits_nonzero_when_a_batch_refreshed_nothing(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""A total failure (auth init or every lookup) is a non-zero exit."""
|
||||
msg = "HLTB: rate-limited, nothing refreshed (auth init: HTTP 429)"
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
_stats_with(lambda *_a: record_fatal(msg))
|
||||
assert exc.value.code == 1
|
||||
assert capsys.readouterr().out.rstrip().splitlines()[-1] == msg
|
||||
|
||||
|
||||
def test_stats_resets_the_tally_first(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Failures from an earlier command in the same process are not re-reported."""
|
||||
record_fatal("stale")
|
||||
_stats_with(None)
|
||||
assert "stale" not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_stats_exits_zero_when_another_lookup_succeeded(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""One refreshed game anywhere in the command means old data survived: exit 0."""
|
||||
msg = "HLTB: all 3 lookups failed, nothing refreshed"
|
||||
|
||||
def one_success_then_a_fatal_batch(*_a: object) -> None:
|
||||
record_success()
|
||||
record_fatal(msg)
|
||||
|
||||
_stats_with(one_success_then_a_fatal_batch)
|
||||
assert capsys.readouterr().out.rstrip().splitlines()[-1] == msg
|
||||
|
||||
|
||||
def test_repeated_fatal_lines_print_once() -> None:
|
||||
"""Four batches failing auth the same way give one line, not four."""
|
||||
for _ in range(4):
|
||||
record_fatal("HLTB: rate-limited, nothing refreshed (auth init: timeout)")
|
||||
assert failure_summary_lines() == [
|
||||
"HLTB: rate-limited, nothing refreshed (auth init: timeout)"
|
||||
]
|
||||
126
steam_backlog_enforcer/tests/test_hltb_pacing.py
Normal file
126
steam_backlog_enforcer/tests/test_hltb_pacing.py
Normal file
@ -0,0 +1,126 @@
|
||||
"""HLTB requests are paced: one at a time, 1.1 s gap after each request (~55/min).
|
||||
|
||||
HLTB rate-limits a burst after ~60 lookups; a full refresh is ~800 games.
|
||||
Fake transport only: no request ever leaves the process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer import _hltb_pacing
|
||||
from steam_backlog_enforcer._hltb_comp_extract import _fetch_detail_one
|
||||
from steam_backlog_enforcer._hltb_failures import BREAKER_LIMIT, shared_breaker
|
||||
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
|
||||
from steam_backlog_enforcer._hltb_types import MAX_CONCURRENT, _AuthInfo
|
||||
from steam_backlog_enforcer.tests._hltb_fake_transport import (
|
||||
MATCH,
|
||||
NO_MATCH,
|
||||
FakeResp,
|
||||
FakeSession,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import aiohttp
|
||||
|
||||
_INTERVAL = 1.1
|
||||
|
||||
|
||||
class _DetailSession:
|
||||
"""Fake session answering every detail GET with a 404."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.gets = 0
|
||||
|
||||
def get(self, _url: str, *, headers: object) -> FakeResp:
|
||||
del headers
|
||||
self.gets += 1
|
||||
return FakeResp(404)
|
||||
|
||||
|
||||
def _ctx(session: FakeSession) -> _SearchCtx:
|
||||
return _SearchCtx(
|
||||
session=cast("aiohttp.ClientSession", session),
|
||||
search_url="https://example.invalid/api/finder",
|
||||
headers={},
|
||||
cache={},
|
||||
auth=_AuthInfo("t"),
|
||||
counter={"done": 0, "found": 0, "failed": 0},
|
||||
)
|
||||
|
||||
|
||||
def _record_sleeps(sem: asyncio.Semaphore) -> tuple[list[float], object]:
|
||||
"""Fake ``asyncio.sleep`` that records each pause and that a slot is held."""
|
||||
pauses: list[float] = []
|
||||
|
||||
async def fake_sleep(seconds: float) -> None:
|
||||
assert sem.locked(), "the pause must hold the request slot"
|
||||
pauses.append(seconds)
|
||||
|
||||
return pauses, fake_sleep
|
||||
|
||||
|
||||
def test_one_request_in_flight() -> None:
|
||||
"""HLTB sees one request in flight at a time."""
|
||||
assert MAX_CONCURRENT == 1
|
||||
|
||||
|
||||
def test_default_interval_stays_under_the_limit() -> None:
|
||||
"""1.1 s between requests keeps us at ~55/min, under the ~60/min limit."""
|
||||
assert _hltb_pacing.REQUEST_INTERVAL_S == _INTERVAL
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [MATCH, NO_MATCH])
|
||||
def test_each_search_post_is_followed_by_a_pause(body: dict[str, object]) -> None:
|
||||
"""Every search POST, match or not, waits the interval inside its slot."""
|
||||
session = FakeSession({"*": FakeResp(200, body)})
|
||||
ctx = _ctx(session)
|
||||
|
||||
async def run() -> list[float]:
|
||||
sem = asyncio.Semaphore(1)
|
||||
pauses, fake_sleep = _record_sleeps(sem)
|
||||
with patch.object(_hltb_pacing, "_sleep", fake_sleep):
|
||||
await _search_one(sem, ctx, 1, "Vox Populi: Poland 2023")
|
||||
return pauses
|
||||
|
||||
pauses = asyncio.run(run())
|
||||
assert len(session.posts) >= 1
|
||||
assert pauses == [_INTERVAL] * len(session.posts)
|
||||
|
||||
|
||||
def test_skipped_lookup_after_breaker_does_not_pause() -> None:
|
||||
"""Once the breaker has tripped nothing is sent, so nothing waits."""
|
||||
session = FakeSession({"*": FakeResp(200, MATCH)})
|
||||
ctx = _ctx(session)
|
||||
ctx.breaker = shared_breaker()
|
||||
for _ in range(BREAKER_LIMIT):
|
||||
ctx.breaker.note(429)
|
||||
|
||||
async def run() -> list[float]:
|
||||
sem = asyncio.Semaphore(1)
|
||||
pauses, fake_sleep = _record_sleeps(sem)
|
||||
with patch.object(_hltb_pacing, "_sleep", fake_sleep):
|
||||
await _search_one(sem, ctx, 1, "Celeste")
|
||||
return pauses
|
||||
|
||||
assert asyncio.run(run()) == []
|
||||
assert session.posts == []
|
||||
|
||||
|
||||
def test_each_detail_get_is_followed_by_a_pause() -> None:
|
||||
"""Detail-page GETs are paced the same way as search POSTs."""
|
||||
session = _DetailSession()
|
||||
|
||||
async def run() -> list[float]:
|
||||
sem = asyncio.Semaphore(1)
|
||||
pauses, fake_sleep = _record_sleeps(sem)
|
||||
with patch.object(_hltb_pacing, "_sleep", fake_sleep):
|
||||
await _fetch_detail_one(sem, cast("aiohttp.ClientSession", session), 77)
|
||||
return pauses
|
||||
|
||||
assert asyncio.run(run()) == [_INTERVAL]
|
||||
assert session.gets == 1
|
||||
@ -0,0 +1,75 @@
|
||||
"""Refetch paths must not lose polls/count_comp when the refetch writes nothing.
|
||||
|
||||
The fetch itself is stubbed to a no-op, which is exactly what a failed HLTB
|
||||
lookup now does: it leaves the in-memory caches untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
from steam_backlog_enforcer import _hltb_confidence, _hltb_types, _scanning_confidence
|
||||
from steam_backlog_enforcer.steam_api import GameInfo
|
||||
|
||||
_ENTRY = {
|
||||
"hours": 20.0,
|
||||
"polls": 100,
|
||||
"count_comp": 500,
|
||||
"rush_hours": 15.0,
|
||||
"leisure_100h": 30.0,
|
||||
"hltb_game_id": 77,
|
||||
}
|
||||
|
||||
|
||||
def _seed(entries: dict[int, dict[str, float]]) -> None:
|
||||
_hltb_types.HLTB_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
_hltb_types.HLTB_CACHE_FILE.write_text(
|
||||
json.dumps({str(k): v for k, v in entries.items()}), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _raw() -> dict[str, dict[str, float]]:
|
||||
data: dict[str, dict[str, float]] = json.loads(
|
||||
_hltb_types.HLTB_CACHE_FILE.read_text(encoding="utf-8")
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def test_refetch_poll_counts_keeps_polls_on_failure() -> None:
|
||||
"""refetch_poll_counts used to drop the entry and restore only its hours."""
|
||||
_seed({1: dict(_ENTRY)})
|
||||
with patch.object(_hltb_confidence, "fetch_hltb_confidence"):
|
||||
polls = _hltb_confidence.refetch_poll_counts([(1, "Game")])
|
||||
assert polls[1] == 100
|
||||
assert _raw()["1"] == _ENTRY
|
||||
|
||||
|
||||
def test_confidence_refresh_keeps_entry_on_failure() -> None:
|
||||
"""The candidate confidence refresh popped hours/polls/count_comp to disk."""
|
||||
_seed({1: dict(_ENTRY)})
|
||||
game = GameInfo(
|
||||
app_id=1,
|
||||
name="Game",
|
||||
total_achievements=10,
|
||||
unlocked_achievements=0,
|
||||
playtime_minutes=0,
|
||||
)
|
||||
with patch.object(_hltb_confidence, "fetch_hltb_confidence"):
|
||||
_scanning_confidence._refresh_candidate_confidence_batch([game], force=True)
|
||||
assert _raw()["1"] == _ENTRY
|
||||
assert (game.comp_100_count, game.count_comp) == (100, 500)
|
||||
|
||||
|
||||
def test_detail_missing_keeps_known_miss_on_failure() -> None:
|
||||
"""A cached miss (-1) must not vanish from the cache when the refetch fails."""
|
||||
miss = dict(
|
||||
_ENTRY, hours=-1.0, rush_hours=-1.0, leisure_100h=-1.0, polls=0, count_comp=0
|
||||
)
|
||||
_seed({1: miss, 2: dict(_ENTRY, rush_hours=-1.0)})
|
||||
with patch.object(_hltb_confidence, "fetch_hltb_times_timed", return_value=0.0):
|
||||
_hltb_confidence.fetch_hltb_detail_missing([(1, "Gone"), (2, "Known")])
|
||||
raw = _raw()
|
||||
assert raw["1"]["hours"] == -1
|
||||
assert raw["2"]["hours"] == 20.0
|
||||
assert raw["2"]["polls"] == 100
|
||||
@ -54,7 +54,7 @@ def _make_ctx(
|
||||
search_url="https://example.com/search",
|
||||
headers={},
|
||||
cache=cache if cache is not None else {},
|
||||
counter={"done": 0, "found": 0},
|
||||
counter={"done": 0, "found": 0, "failed": 0},
|
||||
total=1,
|
||||
progress_cb=progress_cb,
|
||||
)
|
||||
@ -118,6 +118,8 @@ class TestSearchOne:
|
||||
ctx = _make_ctx(session)
|
||||
result = asyncio.run(_search_one(asyncio.Semaphore(1), ctx, 440, "TF2"))
|
||||
assert result is None
|
||||
assert 440 not in ctx.cache # a failed lookup is not a miss
|
||||
assert ctx.counter["failed"] == 1
|
||||
|
||||
def test_non_200(self) -> None:
|
||||
"""Test non 200."""
|
||||
@ -125,6 +127,8 @@ class TestSearchOne:
|
||||
ctx = _make_ctx(_make_session(resp))
|
||||
result = asyncio.run(_search_one(asyncio.Semaphore(1), ctx, 440, "TF2"))
|
||||
assert result is None
|
||||
assert 440 not in ctx.cache # a failed lookup is not a miss
|
||||
assert ctx.counter["failed"] == 1
|
||||
|
||||
def test_fallback_name_without_year_suffix(self) -> None:
|
||||
"""Test fallback name without year suffix."""
|
||||
|
||||
124
steam_backlog_enforcer/tests/test_hltb_search_batch_failures.py
Normal file
124
steam_backlog_enforcer/tests/test_hltb_search_batch_failures.py
Normal file
@ -0,0 +1,124 @@
|
||||
"""Batch-level HLTB failure handling: auth init, total failure, circuit breaker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import patch
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer import _hltb_search
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
HLTBUnavailableError,
|
||||
failure_summary_lines,
|
||||
reset_failures,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_search import _search_batch
|
||||
from steam_backlog_enforcer._hltb_types import _HLTBExtras
|
||||
from steam_backlog_enforcer.tests._hltb_fake_transport import (
|
||||
MATCH,
|
||||
FakeInitSession,
|
||||
FakeResp,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_tally() -> None:
|
||||
reset_failures()
|
||||
|
||||
|
||||
def _batch(
|
||||
status: int,
|
||||
routes: dict[str, object],
|
||||
games: list[tuple[int, str]],
|
||||
fake: FakeInitSession | None = None,
|
||||
) -> FakeInitSession:
|
||||
fake = fake or FakeInitSession(status, routes)
|
||||
with (
|
||||
patch.object(
|
||||
_hltb_search, "_get_hltb_search_url", return_value="https://x.invalid/f"
|
||||
),
|
||||
patch.object(_hltb_search.aiohttp, "ClientSession", lambda **_k: fake),
|
||||
patch.object(_hltb_search.aiohttp, "TCPConnector", lambda **_k: None),
|
||||
):
|
||||
asyncio.run(_search_batch(games, {}, {}, None, _HLTBExtras()))
|
||||
return fake
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"init", [429, aiohttp.ClientConnectionError("no network")], ids=["429", "offline"]
|
||||
)
|
||||
def test_auth_init_failure_fails_the_batch_up_front(
|
||||
init: int | BaseException,
|
||||
) -> None:
|
||||
"""A failed /init means nothing can refresh: raise instead of returning []."""
|
||||
with pytest.raises(HLTBUnavailableError, match="rate-limited, nothing refreshed"):
|
||||
_batch(init, {"*": FakeResp(200, MATCH)}, [(1, "Vox Populi")])
|
||||
assert any("nothing refreshed" in ln for ln in failure_summary_lines())
|
||||
|
||||
|
||||
def test_empty_batch_is_not_a_failure() -> None:
|
||||
"""No games: no request, no error."""
|
||||
fake = _batch(429, {}, [])
|
||||
assert fake.search.posts == []
|
||||
assert failure_summary_lines() == []
|
||||
|
||||
|
||||
def test_breaker_trips_after_five_consecutive_429s() -> None:
|
||||
"""After 5 straight 429s the rest of the batch sends no request at all."""
|
||||
fake = FakeInitSession(200, {"*": FakeResp(429)})
|
||||
games = [(aid, "Celeste") for aid in range(10, 18)]
|
||||
with (
|
||||
patch.object(
|
||||
_hltb_search, "_get_hltb_search_url", return_value="https://x.invalid/f"
|
||||
),
|
||||
patch.object(_hltb_search.aiohttp, "ClientSession", lambda **_k: fake),
|
||||
patch.object(_hltb_search.aiohttp, "TCPConnector", lambda **_k: None),
|
||||
pytest.raises(HLTBUnavailableError, match="all 8 lookups failed"),
|
||||
):
|
||||
asyncio.run(_search_batch(games, {}, {}, None, _HLTBExtras()))
|
||||
assert len(fake.search.posts) == 5
|
||||
lines = failure_summary_lines()
|
||||
assert lines[0] == (
|
||||
"HLTB: 8 lookups failed (HTTP 429 x5, skipped after rate-limit x3),"
|
||||
" cache kept for those games"
|
||||
)
|
||||
assert "HLTB: rate-limited, 8 games not refreshed" in lines
|
||||
|
||||
|
||||
def test_batch_where_every_lookup_failed_raises() -> None:
|
||||
"""Every lookup failing is a total failure."""
|
||||
with pytest.raises(HLTBUnavailableError, match="all 2 lookups failed"):
|
||||
_batch(200, {"*": FakeResp(500)}, [(1, "Celeste"), (2, "Celeste")])
|
||||
|
||||
|
||||
def test_batch_with_one_success_does_not_raise() -> None:
|
||||
"""At least one success: the batch completes and failures are only reported."""
|
||||
_batch(
|
||||
200,
|
||||
{"Vox Populi": FakeResp(200, MATCH), "*": FakeResp(500)},
|
||||
[(1, "Vox Populi"), (2, "Celeste")],
|
||||
)
|
||||
assert failure_summary_lines()[0].startswith("HLTB: 1 lookups failed (HTTP 500 x1)")
|
||||
|
||||
|
||||
def test_tripped_breaker_skips_later_batches_of_the_same_command() -> None:
|
||||
"""Once HLTB rate-limits us, later batches send no request (not even /init)."""
|
||||
with pytest.raises(HLTBUnavailableError):
|
||||
_batch(200, {"*": FakeResp(429)}, [(aid, "Celeste") for aid in range(5)])
|
||||
fake = FakeInitSession(200, {"*": FakeResp(200, MATCH)})
|
||||
with pytest.raises(HLTBUnavailableError, match="rate-limited"):
|
||||
_batch(200, {}, [(1, "Vox"), (2, "Vox")], fake)
|
||||
assert (fake.gets, fake.search.posts) == (0, [])
|
||||
# Games 1 and 2 were in both batches: count games, not lookups.
|
||||
assert "HLTB: rate-limited, 5 games not refreshed" in failure_summary_lines()
|
||||
|
||||
|
||||
def test_reset_clears_a_tripped_breaker() -> None:
|
||||
"""A new command starts with HLTB assumed reachable again."""
|
||||
with pytest.raises(HLTBUnavailableError):
|
||||
_batch(200, {"*": FakeResp(429)}, [(aid, "Celeste") for aid in range(5)])
|
||||
reset_failures()
|
||||
fake = _batch(200, {"*": FakeResp(200, MATCH)}, [(1, "Vox Populi")])
|
||||
assert len(fake.search.posts) == 1
|
||||
@ -0,0 +1,124 @@
|
||||
"""A failed HLTB search must keep the cached entry; only a clean no-match is a miss.
|
||||
|
||||
Fake transport only: no request ever leaves the process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import cast
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer._hltb_failures import (
|
||||
failure_summary_lines,
|
||||
reset_failures,
|
||||
)
|
||||
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
|
||||
from steam_backlog_enforcer._hltb_types import _AuthInfo, _HLTBExtras
|
||||
from steam_backlog_enforcer.tests._hltb_fake_transport import (
|
||||
MATCH,
|
||||
NO_MATCH,
|
||||
FakeResp,
|
||||
FakeSession,
|
||||
)
|
||||
|
||||
|
||||
def _ctx(session: FakeSession, total: int = 1) -> _SearchCtx:
|
||||
return _SearchCtx(
|
||||
session=cast("aiohttp.ClientSession", session),
|
||||
search_url="https://example.invalid/api/finder",
|
||||
headers={},
|
||||
cache={1: 50.0},
|
||||
polls={1: 120},
|
||||
extras=_HLTBExtras(count_comp={1: 900}),
|
||||
auth=_AuthInfo("t"),
|
||||
counter={"done": 0, "found": 0, "failed": 0},
|
||||
total=total,
|
||||
)
|
||||
|
||||
|
||||
def _run(ctx: _SearchCtx, app_id: int, name: str) -> None:
|
||||
asyncio.run(_search_one(asyncio.Semaphore(1), ctx, app_id, name))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_tally() -> None:
|
||||
reset_failures()
|
||||
|
||||
|
||||
_FAILURES = {
|
||||
"429": FakeResp(429),
|
||||
"500": FakeResp(500),
|
||||
"403": FakeResp(403),
|
||||
"timeout": TimeoutError(),
|
||||
"client_error": aiohttp.ClientConnectionError("refused"),
|
||||
"bad_json": FakeResp(200, None),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", sorted(_FAILURES))
|
||||
def test_failed_search_keeps_prior_entry(kind: str) -> None:
|
||||
"""Any non-clean answer leaves hours, polls and count_comp untouched."""
|
||||
ctx = _ctx(FakeSession({"*": _FAILURES[kind]}))
|
||||
_run(ctx, 1, "Celeste")
|
||||
assert ctx.cache[1] == 50.0
|
||||
assert ctx.polls[1] == 120
|
||||
assert ctx.extras.count_comp[1] == 900
|
||||
assert ctx.counter["failed"] == 1
|
||||
assert ctx.counter["found"] == 0
|
||||
|
||||
|
||||
def test_failed_variant_blocks_a_miss() -> None:
|
||||
"""One variant no-match + one variant 429 is a failure, not a miss."""
|
||||
routes = {"Vox Populi Poland 2023": FakeResp(200, NO_MATCH), "*": FakeResp(429)}
|
||||
ctx = _ctx(FakeSession(routes))
|
||||
_run(ctx, 1, "Vox Populi: Poland 2023")
|
||||
assert (ctx.cache[1], ctx.polls[1], ctx.extras.count_comp[1]) == (50.0, 120, 900)
|
||||
assert ctx.counter["failed"] == 1
|
||||
|
||||
|
||||
def test_match_beats_failed_sibling_variant() -> None:
|
||||
"""A real match replaces the entry even when another variant failed."""
|
||||
routes = {"Vox Populi Poland 2023": FakeResp(429), "*": FakeResp(200, MATCH)}
|
||||
ctx = _ctx(FakeSession(routes))
|
||||
_run(ctx, 1, "Vox Populi: Poland 2023")
|
||||
assert (ctx.cache[1], ctx.polls[1], ctx.extras.count_comp[1]) == (10.0, 42, 300)
|
||||
assert ctx.counter == {"done": 1, "found": 1, "failed": 0}
|
||||
|
||||
|
||||
def test_clean_no_match_still_writes_a_miss() -> None:
|
||||
"""Every variant answered 200 with nothing usable: that is a real miss."""
|
||||
ctx = _ctx(FakeSession({"*": FakeResp(200, NO_MATCH)}))
|
||||
_run(ctx, 1, "Vox Populi: Poland 2023")
|
||||
assert (ctx.cache[1], ctx.polls[1], ctx.extras.count_comp[1]) == (-1, 0, 0)
|
||||
assert ctx.counter["failed"] == 0
|
||||
|
||||
|
||||
def test_failure_without_prior_entry_writes_nothing() -> None:
|
||||
"""An uncached game that fails stays uncached so the next run retries it."""
|
||||
ctx = _ctx(FakeSession({"*": FakeResp(429)}))
|
||||
_run(ctx, 2, "Celeste")
|
||||
assert 2 not in ctx.cache
|
||||
assert 2 not in ctx.polls
|
||||
assert 2 not in ctx.extras.count_comp
|
||||
assert ctx.counter == {"done": 1, "found": 0, "failed": 1}
|
||||
|
||||
|
||||
def test_summary_line_counts_reasons() -> None:
|
||||
"""The end-of-run line names every failure reason with its count."""
|
||||
ctx = _ctx(FakeSession({"*": FakeResp(429)}), total=2)
|
||||
_run(ctx, 1, "Celeste")
|
||||
ctx.session = cast("aiohttp.ClientSession", FakeSession({"*": TimeoutError()}))
|
||||
_run(ctx, 1, "Celeste")
|
||||
assert failure_summary_lines()[0] == (
|
||||
"HLTB: 2 lookups failed (HTTP 429 x1, timeouts x1), cache kept for those games"
|
||||
)
|
||||
|
||||
|
||||
def test_no_failures_no_summary() -> None:
|
||||
"""A clean run prints no failure line."""
|
||||
ctx = _ctx(FakeSession({"*": FakeResp(200, MATCH)}))
|
||||
_run(ctx, 1, "Vox Populi")
|
||||
assert failure_summary_lines() == []
|
||||
@ -6,6 +6,9 @@ import asyncio
|
||||
from typing import TYPE_CHECKING, Any, Self
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
|
||||
from steam_backlog_enforcer._hltb_search import (
|
||||
_fetch_batch,
|
||||
_SearchCtx,
|
||||
@ -73,9 +76,9 @@ class TestFetchBatchHltb:
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
pytest.raises(HLTBUnavailableError, match="nothing refreshed"),
|
||||
):
|
||||
results = asyncio.run(_fetch_batch([(440, "TF2")], {}, {}, None))
|
||||
assert results == []
|
||||
asyncio.run(_fetch_batch([(440, "TF2")], {}, {}, None))
|
||||
|
||||
|
||||
class TestPickBestEntry:
|
||||
|
||||
@ -147,7 +147,9 @@ class TestSearchOneGroup2:
|
||||
# Set done to one less than _SAVE_INTERVAL so it triggers save
|
||||
|
||||
ctx.counter["done"] = _SAVE_INTERVAL - 1
|
||||
with patch("steam_backlog_enforcer._hltb_search.save_hltb_cache") as mock_save:
|
||||
with patch(
|
||||
"steam_backlog_enforcer._hltb_search_one.save_hltb_cache"
|
||||
) as mock_save:
|
||||
asyncio.run(_search_one(asyncio.Semaphore(1), ctx, 440, "TF2"))
|
||||
mock_save.assert_called_once()
|
||||
|
||||
|
||||
@ -57,19 +57,26 @@ class TestBackfillPollsForFinished:
|
||||
json.dumps({"1": {"hours": 2.0, "polls": 0}}), encoding="utf-8"
|
||||
)
|
||||
|
||||
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
|
||||
"""Test fake fetch."""
|
||||
data = json.loads(cache_file.read_text(encoding="utf-8"))
|
||||
def fake_fetch(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Test fake search: write into the caches it is handed."""
|
||||
for aid, _name in games:
|
||||
data[str(aid)] = {"hours": 2.0, "polls": 9}
|
||||
cache_file.write_text(json.dumps(data), encoding="utf-8")
|
||||
return {aid: 2.0 for aid, _ in games}
|
||||
cache[aid], polls[aid], count_comp[aid] = 2.0, 9, 0
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
|
||||
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
|
||||
patch(f"{_CMD}.load_snapshot", return_value=[{"app_id": 1, "name": "G"}]),
|
||||
patch(f"{_CONF}.fetch_hltb_confidence_cached", side_effect=fake_fetch),
|
||||
patch(
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
|
||||
side_effect=fake_fetch,
|
||||
),
|
||||
patch(f"{_CMD}._echo"),
|
||||
):
|
||||
result = _cmd_done._backfill_polls_for_finished(_state([1]))
|
||||
@ -82,19 +89,26 @@ class TestBackfillPollsForFinished:
|
||||
json.dumps({"7": {"hours": 1.0, "polls": 0}}), encoding="utf-8"
|
||||
)
|
||||
|
||||
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
|
||||
"""Test fake fetch."""
|
||||
data = json.loads(cache_file.read_text(encoding="utf-8"))
|
||||
def fake_fetch(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Test fake search: write into the caches it is handed."""
|
||||
for aid, _name in games:
|
||||
data[str(aid)] = {"hours": 1.0, "polls": 4}
|
||||
cache_file.write_text(json.dumps(data), encoding="utf-8")
|
||||
return {aid: 1.0 for aid, _ in games}
|
||||
cache[aid], polls[aid], count_comp[aid] = 1.0, 4, 0
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
|
||||
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
|
||||
patch(f"{_CMD}.load_snapshot", return_value=[{"app_id": 7, "name": "G"}]),
|
||||
patch(f"{_CONF}.fetch_hltb_confidence_cached", side_effect=fake_fetch),
|
||||
patch(
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
|
||||
side_effect=fake_fetch,
|
||||
),
|
||||
patch(f"{_CMD}._echo"),
|
||||
):
|
||||
result = _cmd_done._backfill_polls_for_finished(
|
||||
@ -109,20 +123,26 @@ class TestBackfillPollsForFinished:
|
||||
json.dumps({"3": {"hours": 4.0, "polls": 0}}), encoding="utf-8"
|
||||
)
|
||||
|
||||
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
|
||||
# Simulate a refetch returning a miss (hours -1, polls 0).
|
||||
"""Test fake fetch."""
|
||||
data = json.loads(cache_file.read_text(encoding="utf-8"))
|
||||
def fake_fetch(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Test fake search: a clean miss (hours -1, polls 0)."""
|
||||
for aid, _name in games:
|
||||
data[str(aid)] = {"hours": -1, "polls": 0}
|
||||
cache_file.write_text(json.dumps(data), encoding="utf-8")
|
||||
return {aid: -1 for aid, _ in games}
|
||||
cache[aid], polls[aid], count_comp[aid] = -1, 0, 0
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
|
||||
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
|
||||
patch(f"{_CMD}.load_snapshot", return_value=[{"app_id": 3, "name": "G"}]),
|
||||
patch(f"{_CONF}.fetch_hltb_confidence_cached", side_effect=fake_fetch),
|
||||
patch(
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
|
||||
side_effect=fake_fetch,
|
||||
),
|
||||
patch(f"{_CMD}._echo"),
|
||||
):
|
||||
_cmd_done._backfill_polls_for_finished(_state([3]))
|
||||
|
||||
@ -149,19 +149,23 @@ class TestScanningPollsIntegration:
|
||||
json.dumps({"2": {"hours": 3.0, "polls": 0}}), encoding="utf-8"
|
||||
)
|
||||
|
||||
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
|
||||
"""Test fake fetch."""
|
||||
data = json.loads(cache_file.read_text(encoding="utf-8"))
|
||||
def fake_fetch(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Test fake search: write into the caches it is handed."""
|
||||
for aid, _name in games:
|
||||
data[str(aid)] = {"hours": 3.0, "polls": 8}
|
||||
cache_file.write_text(json.dumps(data), encoding="utf-8")
|
||||
return {aid: 3.0 for aid, _ in games}
|
||||
cache[aid], polls[aid], count_comp[aid] = 3.0, 8, 0
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
|
||||
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
|
||||
patch(
|
||||
f"{_CONF}.fetch_hltb_confidence_cached",
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
|
||||
side_effect=fake_fetch,
|
||||
),
|
||||
):
|
||||
@ -186,19 +190,23 @@ class TestScanningPollsIntegration:
|
||||
json.dumps({"2": {"hours": 9.0, "polls": 0}}), encoding="utf-8"
|
||||
)
|
||||
|
||||
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
|
||||
"""Test fake fetch."""
|
||||
data = json.loads(cache_file.read_text(encoding="utf-8"))
|
||||
def fake_fetch(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Test fake search: write into the caches it is handed."""
|
||||
for aid, _name in games:
|
||||
data[str(aid)] = {"hours": -1, "polls": 0}
|
||||
cache_file.write_text(json.dumps(data), encoding="utf-8")
|
||||
return {aid: -1 for aid, _ in games}
|
||||
cache[aid], polls[aid], count_comp[aid] = -1, 0, 0
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
|
||||
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
|
||||
patch(
|
||||
f"{_CONF}.fetch_hltb_confidence_cached",
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
|
||||
side_effect=fake_fetch,
|
||||
),
|
||||
):
|
||||
|
||||
@ -76,7 +76,9 @@ class TestScanningPollsIntegrationGroup2Group2:
|
||||
comp_100_count=3,
|
||||
count_comp=15,
|
||||
)
|
||||
with patch(f"{_SCANCONF}.fetch_hltb_confidence_cached") as mock_fetch:
|
||||
with patch(
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence"
|
||||
) as mock_fetch:
|
||||
_scanning_confidence._refresh_candidate_confidence(game)
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
@ -99,18 +101,23 @@ class TestScanningPollsIntegrationGroup2Group2:
|
||||
count_comp=0,
|
||||
)
|
||||
|
||||
def fake_fetch(_games: list[tuple[int, str]]) -> dict[int, float]:
|
||||
"""Test fake fetch."""
|
||||
data = json.loads(cache_file.read_text(encoding="utf-8"))
|
||||
data["1"] = {"hours": 4.0, "polls": 3, "count_comp": 15}
|
||||
cache_file.write_text(json.dumps(data), encoding="utf-8")
|
||||
return {1: 4.0}
|
||||
def fake_fetch(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Test fake search: write into the caches it is handed."""
|
||||
del games
|
||||
cache[1], polls[1], count_comp[1] = 4.0, 3, 15
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
|
||||
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
|
||||
patch(
|
||||
f"{_SCANCONF}.fetch_hltb_confidence_cached",
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
|
||||
side_effect=fake_fetch,
|
||||
),
|
||||
patch(f"{_SCANCONF}._echo"),
|
||||
@ -151,20 +158,24 @@ class TestScanningPollsIntegrationGroup2Group2:
|
||||
count_comp=0,
|
||||
)
|
||||
|
||||
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
|
||||
"""Test fake fetch."""
|
||||
def fake_fetch(
|
||||
games: list[tuple[int, str]],
|
||||
cache: dict[int, float],
|
||||
polls: dict[int, int],
|
||||
count_comp: dict[int, int],
|
||||
**_kw: object,
|
||||
) -> list[object]:
|
||||
"""Test fake search: write into the caches it is handed."""
|
||||
assert sorted(games) == [(1, "A"), (2, "B")]
|
||||
data = json.loads(cache_file.read_text(encoding="utf-8"))
|
||||
data["1"] = {"hours": 4.0, "polls": 3, "count_comp": 15}
|
||||
data["2"] = {"hours": 5.0, "polls": 3, "count_comp": 15}
|
||||
cache_file.write_text(json.dumps(data), encoding="utf-8")
|
||||
return {1: 4.0, 2: 5.0}
|
||||
cache[1], polls[1], count_comp[1] = 4.0, 3, 15
|
||||
cache[2], polls[2], count_comp[2] = 5.0, 3, 15
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
|
||||
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
|
||||
patch(
|
||||
f"{_SCANCONF}.fetch_hltb_confidence_cached",
|
||||
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
|
||||
side_effect=fake_fetch,
|
||||
) as mock_fetch,
|
||||
patch(f"{_SCANCONF}._echo"),
|
||||
|
||||
@ -52,7 +52,7 @@ class TestConfidenceHelpers:
|
||||
game.comp_100_count = 3
|
||||
game.count_comp = 15
|
||||
with patch(
|
||||
"steam_backlog_enforcer._scanning_confidence.fetch_hltb_confidence_cached",
|
||||
"steam_backlog_enforcer._scanning_confidence.refresh_confidence",
|
||||
) as mock_fetch:
|
||||
_refresh_candidate_confidence_batch([game], force=False)
|
||||
mock_fetch.assert_not_called()
|
||||
@ -62,32 +62,17 @@ class TestConfidenceHelpers:
|
||||
game = _game(app_id=30, name="C", hours=9.5)
|
||||
game.comp_100_count = 0
|
||||
game.count_comp = 0
|
||||
with (
|
||||
patch(
|
||||
"steam_backlog_enforcer._scanning_confidence.load_hltb_cache",
|
||||
side_effect=[{30: 9.5}, {30: -1.0}],
|
||||
),
|
||||
patch(
|
||||
"steam_backlog_enforcer._scanning_confidence.load_hltb_polls_cache",
|
||||
return_value={30: 0},
|
||||
),
|
||||
patch(
|
||||
"steam_backlog_enforcer._scanning_confidence.load_hltb_count_comp_cache",
|
||||
return_value={30: 0},
|
||||
),
|
||||
patch(
|
||||
"steam_backlog_enforcer._scanning_confidence.fetch_hltb_confidence_cached",
|
||||
return_value={30: -1.0},
|
||||
),
|
||||
patch(
|
||||
"steam_backlog_enforcer._scanning_confidence.save_hltb_cache",
|
||||
) as mock_save,
|
||||
):
|
||||
# refresh_confidence owns the hours-preservation rule (see
|
||||
# test_hltb_refetch_failure_keeps_cache); this layer only applies polls.
|
||||
with patch(
|
||||
"steam_backlog_enforcer._scanning_confidence.refresh_confidence",
|
||||
return_value=({30: 9.5}, {30: 4}, {30: 7}),
|
||||
) as mock_refresh:
|
||||
_refresh_candidate_confidence_batch([game], force=True)
|
||||
|
||||
mock_refresh.assert_called_once_with([(30, "C")])
|
||||
assert game.completionist_hours == 9.5
|
||||
saved_cache = mock_save.call_args.args[0]
|
||||
assert saved_cache[30] == 9.5
|
||||
assert (game.comp_100_count, game.count_comp) == (4, 7)
|
||||
|
||||
def test_filter_hltb_confident_candidates_skips_low_confidence(self) -> None:
|
||||
"""Test filter hltb confident candidates skips low confidence."""
|
||||
|
||||
730
web/package-lock.json
generated
730
web/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -14,13 +14,13 @@
|
||||
"coverage": "vitest run --coverage"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tanstack/react-query": "5.105.1",
|
||||
"@tanstack/react-query": "5.107.1",
|
||||
"@tanstack/react-router": "1.170.46",
|
||||
"react": "19.3.1",
|
||||
"react-dom": "19.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "8.0.10",
|
||||
"@babel/core": "8.1.0",
|
||||
"@eslint/js": "10.0.1",
|
||||
"@rolldown/plugin-babel": "0.3.0",
|
||||
"@testing-library/dom": "10.4.2",
|
||||
@ -30,17 +30,17 @@
|
||||
"@types/node": "26.6.6",
|
||||
"@types/react": "19.3.0",
|
||||
"@types/react-dom": "19.3.0",
|
||||
"@vitejs/plugin-react": "6.1.2",
|
||||
"@vitest/coverage-v8": "5.0.5",
|
||||
"@vitejs/plugin-react": "6.2.1",
|
||||
"@vitest/coverage-v8": "5.1.0",
|
||||
"babel-plugin-react-compiler": "1.0.0",
|
||||
"eslint": "10.14.0",
|
||||
"eslint-plugin-react-hooks": "7.1.1",
|
||||
"eslint-plugin-react-refresh": "0.5.7",
|
||||
"globals": "17.14.0",
|
||||
"jsdom": "30.1.3",
|
||||
"jsdom": "31.0.0",
|
||||
"typescript": "6.0.3",
|
||||
"typescript-eslint": "8.72.1",
|
||||
"vite": "8.3.5",
|
||||
"vitest": "5.0.5"
|
||||
"vite": "8.3.6",
|
||||
"vitest": "5.1.0"
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user