HLTB rate limit (429) wipes confidence data in hl…
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (mypy, mypy, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (pylint, pylint, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (rest, , mypy,pylint) (push) Waiting to run
Tests / test (push) Waiting to run
Tests / web-coverage (push) Waiting to run
repo-contract / repo contract (push) Waiting to run

Todo: 9b9342ba
This commit is contained in:
Krzysztof Rudnicki 2026-11-09 08:02:55 +01:00
parent 2ec53d977d
commit 8cc31a2c87
35 changed files with 2236 additions and 1073 deletions

View File

@ -4,7 +4,7 @@
aiohttp==3.14.5
bandit==1.9.4
codespell==2.4.3
coverage==7.17.0
coverage==7.17.2
earned-time @ git+https://github.com/kuhyx/utils@earned-time-v0.21.0#subdirectory=earned_time
howlongtobeatpy==1.0.24
leisure-watch @ git+https://github.com/kuhyx/leisure-watch@leisure-watch-v0.2.7
@ -14,11 +14,11 @@ pre-commit==4.7.0
pylint==4.1.3
pytest==9.2.0
pytest-cov==7.1.0
pytest-randomly==5.0.0
pytest-randomly==5.1.0
pytest-sugar==1.1.1
pytest-timeout==2.4.0
pytest-xdist==3.8.0
requests==2.34.2
ruff==0.17.3
ruff==0.17.4
types-requests==2.33.0.20261030
websockets==17.4

View File

@ -0,0 +1,62 @@
"""Renew the HLTB auth token during a long, paced search batch.
The token from ``/api/search/site/init`` stops working a few minutes after it
is issued: on 2026-11-08 every search ~3 min in came back 403 while a fresh
token got 200 for the same game. Unpaced batches finished before that; a
paced ~800-game refresh takes 15+ min, so the token is renewed by age.
"""
from __future__ import annotations
import logging
import time
from typing import TYPE_CHECKING
from steam_backlog_enforcer._hltb_pacing import pause_after_request
from steam_backlog_enforcer._hltb_search_api import _get_auth_info
if TYPE_CHECKING:
from steam_backlog_enforcer._hltb_search_one import _SearchCtx
from steam_backlog_enforcer._hltb_types import _AuthInfo
logger = logging.getLogger(__name__)
# Well inside the observed ~3 min lifetime (still valid at ~146 s).
AUTH_MAX_AGE_S = 90.0
# Indirection so tests can drive a fake clock.
_now = time.monotonic
def apply_auth(headers: dict[str, str], auth: _AuthInfo) -> None:
"""Put *auth* into the search request headers, replacing any old values."""
headers["x-auth-token"] = auth.token
if auth.hp_key:
headers["x-hp-key"] = auth.hp_key
headers["x-hp-val"] = auth.hp_val
else:
headers.pop("x-hp-key", None)
headers.pop("x-hp-val", None)
async def ensure_fresh_auth(ctx: _SearchCtx) -> str | None:
"""Renew ``ctx.auth`` once it is too old.
Returns ``None`` when the auth is usable, else the failure reason. A
failed renewal trips the breaker: the stale token is never used again.
"""
if _now() - ctx.auth_at < AUTH_MAX_AGE_S:
return None
async with ctx.auth_lock:
if _now() - ctx.auth_at < AUTH_MAX_AGE_S:
return None # another lookup renewed it while we waited
failures: list[str] = []
auth = await _get_auth_info(ctx.search_url, ctx.session, failures)
await pause_after_request()
if auth is None:
ctx.breaker.tripped = True
return f"auth refresh {failures[0] if failures else 'no token'}"
ctx.auth = auth
apply_auth(ctx.headers, auth)
ctx.auth_at = _now()
logger.info("HLTB auth token refreshed.")
return None

View File

@ -8,6 +8,7 @@ from __future__ import annotations
import logging
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
from steam_backlog_enforcer._hltb_types import (
ProgressCb,
_HLTBExtras,
@ -83,7 +84,11 @@ def fetch_hltb_times_cached(
report = reporting_progress_cb(
"Fetching HLTB times", len(uncached), progress_cb
)
elapsed = fetch_hltb_times_timed(uncached, cache, polls, report, extras)
try:
elapsed = fetch_hltb_times_timed(uncached, cache, polls, report, extras)
except HLTBUnavailableError:
# Already logged and tallied by the batch; the cache is untouched.
return cache
# Final save.
save_hltb_cache(cache, polls, extras)

View File

@ -13,6 +13,13 @@ from typing import TYPE_CHECKING, Any
import aiohttp
from steam_backlog_enforcer._hltb_failures import (
detail_status_failed,
exc_reason,
record_failure,
status_reason,
)
from steam_backlog_enforcer._hltb_pacing import pause_after_request
from steam_backlog_enforcer._hltb_page_parse import (
_as_positive_int,
_parse_game_page,
@ -110,10 +117,16 @@ async def _fetch_detail_one(
if resp.status == HTTPStatus.OK:
html = await resp.text()
return _parse_game_page(html)
if not detail_status_failed(resp.status):
return None # no such page: a real answer, not a failure
reason = status_reason(resp.status)
except (TimeoutError, aiohttp.ClientError) as exc:
logger.debug(
"HLTB detail fetch failed for game_id=%d: %s",
hltb_game_id,
exc,
)
reason = exc_reason(exc)
finally:
await pause_after_request()
# The caller keeps the cached hours when this returns None.
logger.warning(
"HLTB detail fetch failed for game_id=%d (%s)", hltb_game_id, reason
)
record_failure(f"detail {reason}")
return None

View File

@ -10,6 +10,7 @@ import logging
import time
from steam_backlog_enforcer._hltb_cached import reporting_progress_cb
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
from steam_backlog_enforcer._hltb_types import (
HLTB_BASE_URL,
ProgressCb,
@ -38,28 +39,36 @@ logger = logging.getLogger(__name__)
# ──────────────────────────────────────────────────────────────
def refetch_poll_counts(missing: list[tuple[int, str]]) -> dict[int, int]:
"""Re-search ``missing`` for poll counts and return the refreshed polls cache.
def refresh_confidence(
games: list[tuple[int, str]],
) -> tuple[dict[int, float], dict[int, int], dict[int, int]]:
"""Re-search ``games`` (cached or not) and save hours, polls and count_comp.
The search-level fetch skips anything already in the hours cache, so the
hours entries are dropped first to force it -- then put back where the
refetch found nothing better, since a prior leisure/DLC estimate is
trusted over a miss.
Nothing is dropped from the cache first: a failed lookup writes nothing,
so the old entry survives. Prior hours beat a clean miss, since a prior
leisure/DLC estimate is trusted over "not found". When HLTB is unavailable
the cache is returned unchanged and nothing is saved.
Returns:
The (hours, polls, count_comp) caches after the refresh.
"""
cache = load_hltb_cache()
polls = load_hltb_polls_cache()
preserved_hours = {aid: cache[aid] for aid, _ in missing if aid in cache}
for aid, _name in missing:
cache.pop(aid, None)
save_hltb_cache(cache, polls)
count_comp = load_hltb_count_comp_cache()
prior_hours = {aid: cache[aid] for aid, _ in games if aid in cache}
try:
fetch_hltb_confidence(games, cache=cache, polls=polls, count_comp=count_comp)
except HLTBUnavailableError:
# Already logged and tallied by the batch; the cache is untouched.
return cache, polls, count_comp
restore_prior_hours(cache, prior_hours)
save_hltb_cache(cache, polls, _HLTBExtras(count_comp=count_comp))
return cache, polls, count_comp
fetch_hltb_confidence_cached(missing)
refreshed_hours = load_hltb_cache()
refreshed_polls = load_hltb_polls_cache()
restore_prior_hours(refreshed_hours, preserved_hours)
save_hltb_cache(refreshed_hours, refreshed_polls)
return refreshed_polls
def refetch_poll_counts(missing: list[tuple[int, str]]) -> dict[int, int]:
"""Re-search ``missing`` for poll counts and return the refreshed polls cache."""
return refresh_confidence(missing)[1]
def fetch_hltb_confidence_cached(
@ -79,15 +88,19 @@ def fetch_hltb_confidence_cached(
len(games) - len(uncached),
)
t0 = time.monotonic()
fetch_hltb_confidence(
uncached,
cache=cache,
polls=polls,
progress_cb=reporting_progress_cb(
"Fetching HLTB poll counts", len(uncached), progress_cb
),
count_comp=count_comp,
)
try:
fetch_hltb_confidence(
uncached,
cache=cache,
polls=polls,
progress_cb=reporting_progress_cb(
"Fetching HLTB poll counts", len(uncached), progress_cb
),
count_comp=count_comp,
)
except HLTBUnavailableError:
# Already logged and tallied by the batch; the cache is untouched.
return cache
elapsed = time.monotonic() - t0
save_hltb_cache(cache, polls, _HLTBExtras(count_comp=count_comp))
@ -116,9 +129,9 @@ def fetch_hltb_detail_missing(
Also backfills ``hltb_game_id`` for any game that already has rush/leisure
data but whose HLTB game ID was never stored (e.g. from an old cache).
Games with both rush data and a game_id are skipped entirely. For the
rest, temporarily removes them from the hours cache so ``fetch_hltb_times``
will visit their detail pages. Restores prior hours for any game the
re-fetch doesn't find.
rest, ``fetch_hltb_times`` re-searches them and visits their detail pages
(it never skips cached games). Prior hours win over a clean miss; a failed
lookup writes nothing, so its old entry survives untouched.
Args:
games: list of (app_id, name) tuples to check.
@ -152,10 +165,7 @@ def fetch_hltb_detail_missing(
hltb_game_id=game_id_cache,
)
# Remove from hours cache so fetch_hltb_times will visit the detail page.
prior_hours: dict[int, float] = {}
for app_id, _ in missing:
prior_hours[app_id] = cache.pop(app_id, -1.0)
prior_hours = {aid: cache[aid] for aid, _ in missing if aid in cache}
n_rush = len(missing_rush)
n_id = len(missing_id_only)
@ -173,7 +183,11 @@ def fetch_hltb_detail_missing(
else:
logger.info("Backfilling HLTB game ID for %d game(s)...", n_id)
report = reporting_progress_cb("Fetching HLTB details", len(missing), progress_cb)
elapsed = fetch_hltb_times_timed(missing, cache, polls, report, extras)
try:
elapsed = fetch_hltb_times_timed(missing, cache, polls, report, extras)
except HLTBUnavailableError:
# Already logged and tallied by the batch; the cache is untouched.
return 0
restore_prior_hours(cache, prior_hours)
@ -200,7 +214,11 @@ def get_hltb_submit_url(game_name: str) -> str | None:
URL like ``https://howlongtobeat.com/submit/game/12345``,
or ``None`` if the game wasn't found.
"""
results = fetch_hltb_times([(0, game_name)])
try:
results = fetch_hltb_times([(0, game_name)])
except HLTBUnavailableError:
# Already logged and tallied by the batch; the cache is untouched.
return None
if results and results[0].hltb_game_id:
return f"{HLTB_BASE_URL}/submit/game/{results[0].hltb_game_id}"
return None

View File

@ -0,0 +1,154 @@
"""Failure accounting for HLTB lookups.
A failed request (non-200, timeout, connection error, non-JSON body) is not
evidence that a game is missing from HLTB, so the search path must keep the
cached entry instead of writing a miss. This module tells the two apart, counts
the failures for the end-of-run summary, and holds the per-batch circuit
breaker that stops hammering HLTB once it starts rate-limiting us.
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass, field
from http import HTTPStatus
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Iterable
# Consecutive 429s/403s after which the command sends no more requests.
BREAKER_LIMIT = 5
_BLOCKED = frozenset({HTTPStatus.TOO_MANY_REQUESTS, HTTPStatus.FORBIDDEN})
SKIPPED_AFTER_RATE_LIMIT = "skipped after rate-limit"
# A detail page that does not exist is a real answer, not a failed lookup.
_DETAIL_NOT_A_FAILURE = frozenset({HTTPStatus.OK, HTTPStatus.NOT_FOUND})
_PLURAL = {"timeout": "timeouts", "connection error": "connection errors"}
class HLTBUnavailableError(RuntimeError):
"""Nothing in an HLTB batch could be refreshed; the cache was left alone."""
@dataclass
class Breaker:
"""Circuit breaker on consecutive HTTP 429 / 403 answers.
403 counts too: a rejected token fails every later request the same way,
and on 2026-11-08 one let 44 doomed requests through before a manual stop.
"""
consecutive_blocked: int = 0
tripped: bool = False
def note(self, status: int | None) -> None:
"""Record one request outcome (``None`` = no HTTP status at all)."""
if status in _BLOCKED:
self.consecutive_blocked += 1
if self.consecutive_blocked >= BREAKER_LIMIT:
self.tripped = True
else:
self.consecutive_blocked = 0
@dataclass
class _Tally:
"""Failures since the last :func:`reset_failures` (one CLI command)."""
reasons: Counter[str] = field(default_factory=Counter)
# App ids, so a game skipped by several passes of one command counts once.
breaker_games: set[int] = field(default_factory=set)
fatal: list[str] = field(default_factory=list)
# Lookups that got a real answer (match or clean miss).
successes: int = 0
# Shared by every batch of the command: once HLTB rate-limits us, later
# batches must not start hammering it again.
breaker: Breaker = field(default_factory=Breaker)
_TALLY = _Tally()
def reset_failures() -> None:
"""Start a fresh tally (call at the start of a command)."""
_TALLY.reasons.clear()
_TALLY.breaker_games.clear()
_TALLY.fatal.clear()
_TALLY.successes = 0
_TALLY.breaker = Breaker()
def shared_breaker() -> Breaker:
"""The command-wide circuit breaker (fresh after :func:`reset_failures`)."""
return _TALLY.breaker
def record_success() -> None:
"""Count one lookup that got a real answer from HLTB."""
_TALLY.successes += 1
def record_failure(reason: str) -> None:
"""Count one failed lookup under *reason*."""
_TALLY.reasons[reason] += 1
def record_breaker(app_ids: Iterable[int]) -> None:
"""Remember the games a tripped breaker left unrefreshed."""
_TALLY.breaker_games.update(app_ids)
def record_fatal(message: str) -> None:
"""Remember a batch that refreshed nothing at all."""
_TALLY.fatal.append(message)
def had_total_failure() -> bool:
"""True when some batch failed outright and no lookup at all succeeded."""
return bool(_TALLY.fatal) and not _TALLY.successes
def status_reason(status: int) -> str:
"""Label for a non-200 answer."""
return f"HTTP {status}"
def exc_reason(exc: BaseException) -> str:
"""Label for a request that raised instead of answering cleanly."""
if isinstance(exc, TimeoutError):
return "timeout"
if isinstance(exc, ValueError): # json.JSONDecodeError and friends
return "bad JSON"
return "connection error"
def detail_status_failed(status: int) -> bool:
"""True when a detail-page answer is a failure rather than a real verdict."""
return status not in _DETAIL_NOT_A_FAILURE
def _label(reason: str) -> str:
"""Plural form for the summary (``detail timeout`` -> ``detail timeouts``)."""
for singular, plural in _PLURAL.items():
if reason.endswith(singular):
return reason[: -len(singular)] + plural
return reason
def failure_summary_lines() -> list[str]:
"""One-line summaries of every failure since the last reset (empty if none)."""
lines: list[str] = []
total = sum(_TALLY.reasons.values())
if total:
parts = ", ".join(
f"{_label(reason)} x{count}"
for reason, count in _TALLY.reasons.most_common()
)
lines.append(
f"HLTB: {total} lookups failed ({parts}), cache kept for those games"
)
if _TALLY.breaker_games:
n_games = len(_TALLY.breaker_games)
lines.append(f"HLTB: rate-limited, {n_games} games not refreshed")
lines.extend(dict.fromkeys(_TALLY.fatal)) # each batch's verdict once
return lines

View File

@ -0,0 +1,21 @@
"""Pacing for HLTB requests.
HLTB answers 429 ("Slow your roll!") after about 60 requests in a minute:
unpaced bursts and a 2 req/s run (2026-11-08) both stopped near 60. A full
library refresh is ~800 games, so the single request slot
(``MAX_CONCURRENT``) waits ``REQUEST_INTERVAL_S`` after every request before
the next one goes out: about 55 requests per minute.
"""
from __future__ import annotations
import asyncio
REQUEST_INTERVAL_S = 1.1
# Indirection so tests can drop the wait without touching asyncio itself.
_sleep = asyncio.sleep
async def pause_after_request() -> None:
"""Hold the caller's request slot for the pacing interval."""
await _sleep(REQUEST_INTERVAL_S)

View File

@ -3,147 +3,76 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass, field
from http import HTTPStatus
import logging
import aiohttp
from steam_backlog_enforcer._hltb_auth_refresh import apply_auth
from steam_backlog_enforcer._hltb_detail import (
_fetch_leisure_times,
)
from steam_backlog_enforcer._hltb_matching import (
_build_result_from_best,
_build_search_variants,
from steam_backlog_enforcer._hltb_failures import (
SKIPPED_AFTER_RATE_LIMIT,
HLTBUnavailableError,
record_breaker,
record_failure,
record_fatal,
shared_breaker,
status_reason,
)
from steam_backlog_enforcer._hltb_search_api import (
_build_search_payload,
_collect_candidates,
_get_auth_info,
_get_hltb_search_url,
_pick_best_hltb_entry,
forget_hltb_search_url,
)
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
from steam_backlog_enforcer._hltb_types import (
_SAVE_INTERVAL,
MAX_CONCURRENT,
HLTBResult,
ProgressCb,
_AuthInfo,
_HLTBExtras,
save_hltb_cache,
)
logger = logging.getLogger(__name__)
# When extended entry has ≥ this many times more hours than the exact match,
# prefer it even if its confidence count is lower.
_EXTENDED_DOMINANCE_RATIO = 4.0
# Minimum combined confidence for the dominance path (avoids picking entries
# that have almost no data at all).
_EXTENDED_MIN_CONFIDENCE = 3
# ──────────────────────────────────────────────────────────────
# HLTB API setup (done once, not per-request like the library)
# ──────────────────────────────────────────────────────────────
# ──────────────────────────────────────────────────────────────
# Async fetching with shared session & progress
# ──────────────────────────────────────────────────────────────
@dataclass
class _SearchCtx:
"""Shared context for HLTB search requests."""
session: aiohttp.ClientSession
search_url: str
headers: dict[str, str]
cache: dict[int, float]
polls: dict[int, int] = field(default_factory=dict)
extras: _HLTBExtras = field(default_factory=_HLTBExtras)
auth: _AuthInfo | None = None
counter: dict[str, int] = field(default_factory=dict)
total: int = 0
progress_cb: ProgressCb | None = None
def _fail_batch(message: str) -> HLTBUnavailableError:
"""Log and remember a batch that refreshed nothing; return the error."""
logger.warning("%s", message)
record_fatal(message)
return HLTBUnavailableError(message)
async def _search_one(
sem: asyncio.Semaphore,
ctx: _SearchCtx,
app_id: int,
name: str,
) -> HLTBResult | None:
"""Search HLTB for one game via direct POST, update cache."""
async with sem:
result: HLTBResult | None = None
for query_name in _build_search_variants(name):
payload = _build_search_payload(query_name, ctx.auth)
try:
async with ctx.session.post(
ctx.search_url,
headers=ctx.headers,
data=payload,
) as resp:
if resp.status != HTTPStatus.OK:
continue
data = await resp.json()
candidates = _collect_candidates(query_name, data)
# When we stripped ": subtitle" from the original name to
# get query_name, only keep full-edition entries (those
# whose HLTB name starts with query_name + ":"/"-") or
# exact name/alias matches. This prevents "Vox Populi"
# (stripped from "Vox Populi: Poland 2023") from falsely
# matching "Vox Populi Vox Dei 2".
if ":" in name and ":" not in query_name:
lower_q = query_name.lower()
candidates = [
(e, s)
for e, s in candidates
if (e.get("game_name") or "").lower() == lower_q
or (e.get("game_alias") or "").lower() == lower_q
or (e.get("game_name") or "")
.lower()
.startswith(lower_q + ":")
or (e.get("game_name") or "")
.lower()
.startswith(lower_q + " -")
]
best = _pick_best_hltb_entry(query_name, candidates)
if best is None:
continue
result = _build_result_from_best(app_id, name, query_name, best)
break
except (TimeoutError, aiohttp.ClientError) as exc:
logger.debug("HLTB search failed for '%s': %s", query_name, exc)
def _skip_rate_limited(games: list[tuple[int, str]]) -> HLTBUnavailableError:
"""Count a whole batch as not refreshed because the breaker already tripped."""
for _game in games:
record_failure(SKIPPED_AFTER_RATE_LIMIT)
record_breaker(app_id for app_id, _name in games)
msg = f"HLTB: rate-limited, skipped {len(games)} lookups"
logger.warning("%s", msg)
return HLTBUnavailableError(msg)
# Update cache immediately (miss = -1).
if result is not None:
ctx.cache[app_id] = result.completionist_hours
ctx.polls[app_id] = result.comp_100_count
ctx.extras.count_comp[app_id] = result.count_comp
if result.hltb_game_id > 0:
ctx.extras.hltb_game_id[app_id] = result.hltb_game_id
ctx.counter["found"] += 1
else:
ctx.cache[app_id] = -1
ctx.polls[app_id] = 0
ctx.extras.count_comp[app_id] = 0
ctx.counter["done"] += 1
done = ctx.counter["done"]
# Incremental save every _SAVE_INTERVAL lookups.
if not done % _SAVE_INTERVAL:
save_hltb_cache(ctx.cache, ctx.polls, ctx.extras)
# Report progress.
if ctx.progress_cb is not None:
ctx.progress_cb(done, ctx.total, ctx.counter["found"], name)
return result
def _check_batch(ctx: _SearchCtx) -> None:
"""Report a batch's failures; raise when not a single lookup succeeded."""
failed = ctx.counter["failed"]
if not failed:
return
logger.warning(
"HLTB: %d/%d lookups failed in this batch; cache kept for them",
failed,
ctx.total,
)
if ctx.breaker.tripped:
record_breaker(ctx.failed_ids)
if failed == ctx.total:
msg = f"HLTB: all {failed} lookups failed, nothing refreshed"
raise _fail_batch(msg)
async def _search_batch(
@ -157,7 +86,16 @@ async def _search_batch(
Hours, poll counts and the extras come from the search results alone;
:func:`_fetch_batch` layers the per-game detail pages on top of this.
Raises:
HLTBUnavailableError: auth init failed, or every lookup failed. The
caches are untouched in both cases.
"""
if not games:
return []
breaker = shared_breaker()
if breaker.tripped:
raise _skip_rate_limited(games)
# 1. Discover the search URL (sync, one-time).
search_url = _get_hltb_search_url()
logger.info("HLTB search URL: %s", search_url)
@ -165,12 +103,16 @@ async def _search_batch(
timeout = aiohttp.ClientTimeout(total=20, sock_read=15)
# 2. Get auth info (separate session — avoids reuse issues).
auth_failures: list[str] = []
async with aiohttp.ClientSession(timeout=timeout) as init_session:
auth = await _get_auth_info(search_url, init_session)
auth = await _get_auth_info(search_url, init_session, auth_failures)
if auth is None:
logger.warning("Could not get HLTB auth info, aborting fetch.")
forget_hltb_search_url()
return []
why = auth_failures[0] if auth_failures else "no token"
if why == status_reason(HTTPStatus.TOO_MANY_REQUESTS):
breaker.tripped = True
msg = f"HLTB: rate-limited, nothing refreshed (auth init: {why})"
raise _fail_batch(msg)
logger.info("HLTB auth token acquired.")
# 3. Build shared headers for all search requests.
@ -181,11 +123,8 @@ async def _search_batch(
"Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0"
),
"referer": "https://howlongtobeat.com/",
"x-auth-token": auth.token,
}
if auth.hp_key:
headers["x-hp-key"] = auth.hp_key
headers["x-hp-val"] = auth.hp_val
apply_auth(headers, auth)
# 4. Fire all searches through a single persistent session.
sem = asyncio.Semaphore(MAX_CONCURRENT)
@ -205,12 +144,14 @@ async def _search_batch(
polls=polls,
extras=extras,
auth=auth,
counter={"done": 0, "found": 0},
counter={"done": 0, "found": 0, "failed": 0},
total=len(games),
progress_cb=progress_cb,
breaker=breaker,
)
tasks = [_search_one(sem, ctx, app_id, name) for app_id, name in games]
results = await asyncio.gather(*tasks)
_check_batch(ctx)
return [r for r in results if r is not None]

View File

@ -15,6 +15,7 @@ from typing import Any
import aiohttp
from howlongtobeatpy.HTMLRequests import HTMLRequests
from steam_backlog_enforcer._hltb_failures import exc_reason, status_reason
from steam_backlog_enforcer._hltb_matching import (
_find_best_extended,
_find_exact_match,
@ -91,14 +92,20 @@ def forget_hltb_search_url() -> None:
async def _get_auth_info(
search_url: str,
session: aiohttp.ClientSession,
failures: list[str] | None = None,
) -> _AuthInfo | None:
"""Fetch the HLTB auth token and honeypot key/val (one GET request)."""
"""Fetch the HLTB auth token and honeypot key/val (one GET request).
On failure returns ``None`` and, when *failures* is given, appends why
(``HTTP 429``, ``timeout``, ...) so the caller can report it.
"""
init_url = search_url + "/init"
ts = int(time.time() * 1000)
headers = {
"User-Agent": _USER_AGENT,
"referer": "https://howlongtobeat.com/",
}
reason = "no token"
try:
async with session.get(
init_url,
@ -108,15 +115,19 @@ async def _get_auth_info(
if resp.status == HTTPStatus.OK:
data = await resp.json()
token: str | None = data.get("token")
if token is None:
return None
return _AuthInfo(
token=token,
hp_key=data.get("hpKey", ""),
hp_val=data.get("hpVal", ""),
)
except TimeoutError, aiohttp.ClientError:
logger.warning("Failed to get HLTB auth token")
if token is not None:
return _AuthInfo(
token=token,
hp_key=data.get("hpKey", ""),
hp_val=data.get("hpVal", ""),
)
else:
reason = status_reason(resp.status)
except (TimeoutError, aiohttp.ClientError, ValueError) as exc:
reason = exc_reason(exc)
logger.warning("Failed to get HLTB auth token (%s)", reason)
if failures is not None:
failures.append(reason)
return None

View File

@ -0,0 +1,198 @@
"""One HLTB game lookup: query each name variant, classify, write the cache.
Split out of :mod:`steam_backlog_enforcer._hltb_search` to keep both files
under the 250-line cap. A failed request never writes a miss: only a lookup
whose every variant answered a clean 200 with nothing usable does.
"""
from __future__ import annotations
import asyncio
from dataclasses import dataclass, field
from http import HTTPStatus
import logging
import time
from typing import Any
import aiohttp
from steam_backlog_enforcer._hltb_auth_refresh import ensure_fresh_auth
from steam_backlog_enforcer._hltb_failures import (
SKIPPED_AFTER_RATE_LIMIT,
Breaker,
exc_reason,
record_failure,
record_success,
status_reason,
)
from steam_backlog_enforcer._hltb_matching import (
_build_result_from_best,
_build_search_variants,
)
from steam_backlog_enforcer._hltb_pacing import pause_after_request
from steam_backlog_enforcer._hltb_search_api import (
_build_search_payload,
_collect_candidates,
_pick_best_hltb_entry,
)
from steam_backlog_enforcer._hltb_types import (
_SAVE_INTERVAL,
HLTBResult,
ProgressCb,
_AuthInfo,
_HLTBExtras,
save_hltb_cache,
)
logger = logging.getLogger(__name__)
@dataclass
class _SearchCtx:
"""Shared context for HLTB search requests."""
session: aiohttp.ClientSession
search_url: str
headers: dict[str, str]
cache: dict[int, float]
polls: dict[int, int] = field(default_factory=dict)
extras: _HLTBExtras = field(default_factory=_HLTBExtras)
auth: _AuthInfo | None = None
counter: dict[str, int] = field(default_factory=dict)
total: int = 0
progress_cb: ProgressCb | None = None
breaker: Breaker = field(default_factory=Breaker)
# When ``auth`` was issued (monotonic) and the lock for renewing it.
auth_at: float = field(default_factory=time.monotonic)
auth_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
# Games whose lookup failed (the breaker reports them by id).
failed_ids: set[int] = field(default_factory=set)
# Only flush incrementally once something was actually written.
dirty: bool = False
_Best = tuple[dict[str, Any], float]
def _full_edition_candidates(
name: str, query_name: str, candidates: list[_Best]
) -> list[_Best]:
"""Drop unrelated entries when ``query_name`` is ``name`` minus a subtitle.
When we stripped ": subtitle" from the original name to get query_name,
only keep full-edition entries (those whose HLTB name starts with
query_name + ":"/"-") or exact name/alias matches. This prevents "Vox
Populi" (stripped from "Vox Populi: Poland 2023") from falsely matching
"Vox Populi Vox Dei 2".
"""
if ":" not in name or ":" in query_name:
return candidates
lower_q = query_name.lower()
kept: list[_Best] = []
for entry, sim in candidates:
game_name = (entry.get("game_name") or "").lower()
alias = (entry.get("game_alias") or "").lower()
if lower_q in {game_name, alias} or game_name.startswith(
(lower_q + ":", lower_q + " -")
):
kept.append((entry, sim))
return kept
async def _query_variant(
ctx: _SearchCtx, name: str, query_name: str
) -> tuple[_Best | None, str | None]:
"""Run one search variant.
Returns ``(best, None)`` on a match, ``(None, None)`` on a clean 200 with
no usable entry, and ``(None, reason)`` when the request itself failed.
"""
payload = _build_search_payload(query_name, ctx.auth)
try:
async with ctx.session.post(
ctx.search_url, headers=ctx.headers, data=payload
) as resp:
ctx.breaker.note(resp.status)
if resp.status != HTTPStatus.OK:
return None, status_reason(resp.status)
data = await resp.json()
except (TimeoutError, aiohttp.ClientError, ValueError) as exc:
ctx.breaker.note(None)
return None, exc_reason(exc)
finally:
await pause_after_request()
if not isinstance(data, dict):
return None, "bad JSON"
candidates = _full_edition_candidates(
name, query_name, _collect_candidates(query_name, data)
)
return _pick_best_hltb_entry(query_name, candidates), None
def _record_outcome(
ctx: _SearchCtx, app_id: int, result: HLTBResult | None, failure: str | None
) -> None:
"""Write a match or a clean miss; a failed lookup keeps the old entry."""
if result is not None:
ctx.cache[app_id] = result.completionist_hours
ctx.polls[app_id] = result.comp_100_count
ctx.extras.count_comp[app_id] = result.count_comp
if result.hltb_game_id > 0:
ctx.extras.hltb_game_id[app_id] = result.hltb_game_id
ctx.counter["found"] += 1
ctx.dirty = True
record_success()
elif failure is not None:
ctx.counter["failed"] += 1
ctx.failed_ids.add(app_id)
record_failure(failure)
else:
# Every variant answered cleanly with nothing usable: a real miss.
ctx.cache[app_id] = -1
ctx.polls[app_id] = 0
ctx.extras.count_comp[app_id] = 0
ctx.dirty = True
record_success()
async def _search_one(
sem: asyncio.Semaphore,
ctx: _SearchCtx,
app_id: int,
name: str,
) -> HLTBResult | None:
"""Search HLTB for one game via direct POST, update cache."""
async with sem:
result: HLTBResult | None = None
failure: str | None = None
for query_name in _build_search_variants(name):
if ctx.breaker.tripped:
failure = failure or SKIPPED_AFTER_RATE_LIMIT
break
auth_failure = await ensure_fresh_auth(ctx)
if auth_failure is not None:
failure = failure or auth_failure
break
best, reason = await _query_variant(ctx, name, query_name)
if best is not None:
result = _build_result_from_best(app_id, name, query_name, best)
break
failure = failure or reason
if result is None and failure not in {None, SKIPPED_AFTER_RATE_LIMIT}:
logger.warning(
"HLTB lookup failed for '%s' (%s), cache kept", name, failure
)
_record_outcome(ctx, app_id, result, failure)
ctx.counter["done"] += 1
done = ctx.counter["done"]
# Incremental save every _SAVE_INTERVAL lookups.
if ctx.dirty and not done % _SAVE_INTERVAL:
save_hltb_cache(ctx.cache, ctx.polls, ctx.extras)
# Report progress.
if ctx.progress_cb is not None:
ctx.progress_cb(done, ctx.total, ctx.counter["found"], name)
return result

View File

@ -13,7 +13,8 @@ from steam_backlog_enforcer.config import CONFIG_DIR, _atomic_write
logger = logging.getLogger(__name__)
HLTB_CACHE_FILE = CONFIG_DIR / "hltb_cache.json"
MAX_CONCURRENT = 60 # parallel requests to HLTB
# Requests in flight to HLTB; each slot is paced by _hltb_pacing.
MAX_CONCURRENT = 1
_SAVE_INTERVAL = 50 # flush cache to disk every N results
MIN_SIMILARITY = 0.5
HLTB_BASE_URL = "https://howlongtobeat.com"

View File

@ -5,14 +5,10 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from steam_backlog_enforcer._hltb_confidence import fetch_hltb_confidence_cached
from steam_backlog_enforcer._hltb_confidence import refresh_confidence
from steam_backlog_enforcer._hltb_types import (
_HLTBExtras,
load_hltb_cache,
load_hltb_count_comp_cache,
load_hltb_polls_cache,
restore_prior_hours,
save_hltb_cache,
)
from steam_backlog_enforcer._polls_reporting import (
_backfill_polls_for_finished,
@ -105,27 +101,8 @@ def _refresh_candidate_confidence_batch(
else:
_echo(f" Refreshing HLTB confidence for {len(refresh_slice)} candidate(s)...")
cache = load_hltb_cache()
polls = load_hltb_polls_cache()
count_comp = load_hltb_count_comp_cache()
app_ids = [game.app_id for game in refresh_slice]
names = [(game.app_id, game.name) for game in refresh_slice]
prior_hours = {aid: cache.get(aid, -1) for aid in app_ids}
for aid in app_ids:
cache.pop(aid, None)
polls.pop(aid, None)
count_comp.pop(aid, None)
save_hltb_cache(cache, polls, _HLTBExtras(count_comp=count_comp))
fetch_hltb_confidence_cached(names)
refreshed_hours = load_hltb_cache()
refreshed_polls = load_hltb_polls_cache()
refreshed_count_comp = load_hltb_count_comp_cache()
restore_prior_hours(refreshed_hours, prior_hours)
save_hltb_cache(
refreshed_hours, refreshed_polls, _HLTBExtras(count_comp=refreshed_count_comp)
_, refreshed_polls, refreshed_count_comp = refresh_confidence(
[(game.app_id, game.name) for game in refresh_slice]
)
for game in refresh_slice:

View File

@ -3,8 +3,14 @@
from __future__ import annotations
import logging
import sys
from typing import TYPE_CHECKING
from steam_backlog_enforcer._hltb_failures import (
failure_summary_lines,
had_total_failure,
reset_failures,
)
from steam_backlog_enforcer._hltb_types import (
_read_raw_cache,
)
@ -131,7 +137,22 @@ def cmd_stats(_config: Config, state: State) -> None:
3. Leisure — comp_100_h (slowest 100 %) + DLC leisure per HLTB.
4. Worst — absolute maximum recorded time (any category) per HLTB.
5. Your play style — extrapolated from completed-game calibration vs HLTB.
Ends with one line per kind of HLTB failure (the cache is kept for those
games) and exits 1 when some HLTB batch refreshed nothing at all.
"""
reset_failures()
try:
_print_stats(_config, state)
finally:
for line in failure_summary_lines():
_echo(line)
if had_total_failure():
sys.exit(1)
def _print_stats(_config: Config, state: State) -> None:
"""Gather the stats data and print every scenario for :func:`cmd_stats`."""
games = require_snapshot_games()
if games is None:
return

View File

@ -0,0 +1,93 @@
"""Fake aiohttp transport for HLTB search tests: no request leaves the process."""
from __future__ import annotations
import json
from typing import Any, Self
MATCH = {
"data": [
{
"game_name": "Vox Populi",
"game_alias": "",
"game_type": "game",
"game_id": 77,
"comp_100": 36000,
"comp_100_count": 42,
"count_comp": 300,
}
]
}
NO_MATCH: dict[str, Any] = {"data": []}
class FakeResp:
"""Minimal aiohttp response stand-in."""
def __init__(self, status: int, body: dict[str, Any] | None = None) -> None:
"""Store the canned status and JSON body (``None`` = not JSON)."""
self.status = status
self._body = body
async def __aenter__(self) -> Self:
"""Enter."""
return self
async def __aexit__(self, *args: object) -> None:
"""Exit."""
async def json(self) -> dict[str, Any]:
"""Return the body, or fail like a garbage payload would."""
if self._body is None:
msg = "Expecting value"
raise json.JSONDecodeError(msg, "<html>", 0)
return self._body
class FakeSession:
"""Fake session routing each search POST by its first search term."""
def __init__(self, routes: dict[str, object]) -> None:
"""Map a query's first word to a _Resp or an exception to raise."""
self.routes = routes
self.posts: list[str] = []
def post(self, _url: str, *, headers: object, data: str) -> FakeResp:
"""Return (or raise) the canned outcome for this query."""
del headers
terms = json.loads(data)["searchTerms"]
query = " ".join(terms)
self.posts.append(query)
outcome = self.routes.get(query, self.routes.get("*"))
if isinstance(outcome, BaseException):
raise outcome
assert isinstance(outcome, FakeResp)
return outcome
class FakeInitSession:
"""Fake ClientSession: GET /init answers ``status``; POSTs use ``routes``."""
def __init__(self, status: int | BaseException, routes: dict[str, object]) -> None:
"""Store the canned init outcome and search routes."""
self.status = status
self.search = FakeSession(routes)
self.gets = 0
async def __aenter__(self) -> Self:
"""Enter."""
return self
async def __aexit__(self, *args: object) -> None:
"""Exit."""
def get(self, *_a: object, **_k: object) -> FakeResp:
"""Answer the auth init."""
self.gets += 1
if isinstance(self.status, BaseException):
raise self.status
return FakeResp(self.status, {"token": "tok"})
def post(self, url: str, *, headers: object, data: str) -> FakeResp:
"""Delegate searches."""
return self.search.post(url, headers=headers, data=data)

View File

@ -16,6 +16,7 @@ from unittest.mock import patch
import pytest
from steam_backlog_enforcer._hltb_failures import reset_failures
from steam_backlog_enforcer._hltb_search_api import forget_hltb_search_url
if TYPE_CHECKING:
@ -65,16 +66,8 @@ def _isolate_cover_art(tmp_path: Path) -> Iterator[None]:
yield
@pytest.fixture(autouse=True)
def _forget_hltb_search_url() -> Iterator[None]:
"""Start every test without a remembered HLTB search endpoint.
``_get_hltb_search_url`` memoises a discovered URL for half an hour; a
value left behind by one test would hide the discovery path in the next.
"""
forget_hltb_search_url()
yield
forget_hltb_search_url()
async def _no_pause(_seconds: float) -> None:
"""Stand-in for the HLTB pacing sleep."""
@pytest.fixture(autouse=True)
@ -83,7 +76,13 @@ def _forget_hltb_search_url() -> Iterator[None]:
``_get_hltb_search_url`` memoises a discovered URL for half an hour; a
value left behind by one test would hide the discovery path in the next.
The HLTB failure tally and its shared circuit breaker are reset too: a
breaker tripped by one test would make the next one skip every lookup.
The per-request HLTB pacing pause is dropped so tests never wait on it.
"""
forget_hltb_search_url()
yield
reset_failures()
with patch("steam_backlog_enforcer._hltb_pacing._sleep", _no_pause):
yield
forget_hltb_search_url()
reset_failures()

View File

@ -10,9 +10,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
import aiohttp
from steam_backlog_enforcer._hltb_matching import _similarity
from steam_backlog_enforcer._hltb_search import _AuthInfo
from steam_backlog_enforcer._hltb_search_api import _get_auth_info, _get_hltb_search_url
from steam_backlog_enforcer._hltb_types import (
_AuthInfo,
load_hltb_cache,
save_hltb_cache,
)

View File

@ -0,0 +1,194 @@
"""A long paced batch renews the HLTB token before it expires.
Seen 2026-11-08: at ~55 requests/min every search ~3 min after /init came
back 403 while a fresh token got 200 for the same game. Fake transport and a
fake clock only: no request leaves the process and nothing sleeps.
"""
from __future__ import annotations
import asyncio
import json
from typing import TYPE_CHECKING, Self, cast
from unittest.mock import patch
from steam_backlog_enforcer import _hltb_auth_refresh
from steam_backlog_enforcer._hltb_failures import (
BREAKER_LIMIT,
SKIPPED_AFTER_RATE_LIMIT,
Breaker,
failure_summary_lines,
)
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
from steam_backlog_enforcer._hltb_types import _AuthInfo
from steam_backlog_enforcer.tests._hltb_fake_transport import MATCH, FakeResp
if TYPE_CHECKING:
import aiohttp
_AGE = _hltb_auth_refresh.AUTH_MAX_AGE_S
class _RotatingSession:
"""Fake session: each /init GET hands out a new token; POSTs record auth."""
def __init__(self, init_status: int = 200, post_status: int = 200) -> None:
self.init_status = init_status
self.post_status = post_status
self.gets = 0
# (x-auth-token header, honeypot value in the body) per POST.
self.posts: list[tuple[str | None, str | None]] = []
def get(self, *_a: object, **_k: object) -> FakeResp:
self.gets += 1
n = self.gets
return FakeResp(
self.init_status, {"token": f"tok{n}", "hpKey": "hp", "hpVal": f"v{n}"}
)
def post(self, _url: str, *, headers: dict[str, str], data: str) -> FakeResp:
self.posts.append((headers.get("x-auth-token"), json.loads(data).get("hp")))
body = MATCH if self.post_status == 200 else None
return FakeResp(self.post_status, body)
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *args: object) -> None:
pass
class _ListBodySession(_RotatingSession):
"""Fake session whose searches answer 200 with a JSON list, not an object."""
def post(self, _url: str, *, headers: dict[str, str], data: str) -> FakeResp:
del headers, data
return FakeResp(200, cast("dict[str, object]", []))
class _Clock:
"""Fake monotonic clock advanced by the test."""
def __init__(self) -> None:
self.t = 1000.0
def __call__(self) -> float:
return self.t
def _ctx(session: _RotatingSession, clock: _Clock) -> _SearchCtx:
return _SearchCtx(
session=cast("aiohttp.ClientSession", session),
search_url="https://example.invalid/api/finder",
headers={"x-auth-token": "tok0", "x-hp-key": "hp", "x-hp-val": "v0"},
cache={},
auth=_AuthInfo("tok0", "hp", "v0"),
counter={"done": 0, "found": 0, "failed": 0},
breaker=Breaker(),
auth_at=clock.t,
)
def _lookups(ctx: _SearchCtx, clock: _Clock, steps: list[float]) -> None:
"""Run one lookup per step, advancing the clock by that many seconds first."""
async def run() -> None:
sem = asyncio.Semaphore(1)
for i, step in enumerate(steps):
clock.t += step
await _search_one(sem, ctx, i + 1, "Vox Populi")
with patch.object(_hltb_auth_refresh, "_now", clock):
asyncio.run(run())
def test_young_token_is_not_refreshed() -> None:
"""Within the max age no extra /init request is made."""
session, clock = _RotatingSession(), _Clock()
_lookups(_ctx(session, clock), clock, [0, _AGE / 2])
assert session.gets == 0
assert session.posts == [("tok0", "v0"), ("tok0", "v0")]
def test_aged_token_is_refreshed_into_headers_and_body() -> None:
"""Past the max age the next POST carries the new token and honeypot."""
session, clock = _RotatingSession(), _Clock()
ctx = _ctx(session, clock)
_lookups(ctx, clock, [0, _AGE + 1])
assert session.gets == 1
assert session.posts == [("tok0", "v0"), ("tok1", "v1")]
assert ctx.auth == _AuthInfo("tok1", "hp", "v1")
def test_one_refresh_per_elapsed_interval() -> None:
"""Many lookups inside one interval share a single refresh."""
session, clock = _RotatingSession(), _Clock()
_lookups(_ctx(session, clock), clock, [_AGE + 1, 1, 1, 1, _AGE + 1, 1])
assert session.gets == 2
assert [tok for tok, _hp in session.posts] == [
"tok1",
"tok1",
"tok1",
"tok1",
"tok2",
"tok2",
]
def test_failed_refresh_stops_sending_and_trips_the_breaker() -> None:
"""A refresh that gets 429 never falls back to the stale token."""
session, clock = _RotatingSession(init_status=429), _Clock()
ctx = _ctx(session, clock)
_lookups(ctx, clock, [_AGE + 1, 1, 1])
assert session.gets == 1
assert session.posts == []
assert ctx.breaker.tripped
assert ctx.counter["failed"] == 3
summary = failure_summary_lines()[0]
assert "auth refresh HTTP 429 x1" in summary
assert f"{SKIPPED_AFTER_RATE_LIMIT} x2" in summary
def test_consecutive_403s_trip_the_breaker() -> None:
"""A rejected token cannot turn into hundreds of doomed requests."""
session, clock = _RotatingSession(post_status=403), _Clock()
ctx = _ctx(session, clock)
_lookups(ctx, clock, [0] * (BREAKER_LIMIT + 3))
assert len(session.posts) == BREAKER_LIMIT
assert ctx.breaker.tripped
assert ctx.counter["failed"] == BREAKER_LIMIT + 3
def test_concurrent_lookups_share_one_refresh() -> None:
"""A lookup that waited on the lock reuses the token renewed meanwhile."""
session, clock = _RotatingSession(), _Clock()
ctx = _ctx(session, clock)
clock.t += _AGE + 1
async def yielding_pause(_seconds: float) -> None:
await asyncio.sleep(0) # let the other lookup reach the lock
async def run() -> list[str | None]:
return list(
await asyncio.gather(
_hltb_auth_refresh.ensure_fresh_auth(ctx),
_hltb_auth_refresh.ensure_fresh_auth(ctx),
)
)
with (
patch.object(_hltb_auth_refresh, "_now", clock),
patch("steam_backlog_enforcer._hltb_pacing._sleep", yielding_pause),
):
assert asyncio.run(run()) == [None, None]
assert session.gets == 1
def test_non_object_json_is_a_failure() -> None:
"""A 200 whose body is not a JSON object is "bad JSON", never a miss."""
session, clock = _ListBodySession(), _Clock()
ctx = _ctx(session, clock)
ctx.cache[1] = 50.0
_lookups(ctx, clock, [0])
assert ctx.cache[1] == 50.0
assert "bad JSON x1" in failure_summary_lines()[0]

View File

@ -4,8 +4,8 @@ from __future__ import annotations
import json
from steam_backlog_enforcer._hltb_search import _AuthInfo
from steam_backlog_enforcer._hltb_search_api import _build_search_payload
from steam_backlog_enforcer._hltb_types import _AuthInfo
class TestBuildSearchPayload:

View File

@ -0,0 +1,93 @@
"""A failed HLTB batch leaves every cached entry point's cache alone.
No network: the batch functions are faked at the module boundary.
"""
from __future__ import annotations
from unittest.mock import patch
from steam_backlog_enforcer import _hltb_cached, _hltb_confidence
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
_CONF = "steam_backlog_enforcer._hltb_confidence"
_DOWN = HLTBUnavailableError("HLTB: rate-limited, nothing refreshed")
def _found(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Fake search that finds every game."""
for aid, _name in games:
cache[aid], polls[aid], count_comp[aid] = 5.0, 7, 40
return []
def test_cached_fetch_saves_new_results() -> None:
"""Uncached games are searched and the result is saved."""
with (
patch(f"{_CONF}.fetch_hltb_confidence", side_effect=_found),
patch(f"{_CONF}.save_hltb_cache") as save,
):
result = _hltb_confidence.fetch_hltb_confidence_cached([(1, "Celeste")])
assert result == {1: 5.0}
save.assert_called_once()
def test_cached_fetch_keeps_cache_when_hltb_is_down() -> None:
"""An unavailable batch returns the cache as loaded and saves nothing."""
with (
patch(f"{_CONF}.load_hltb_cache", return_value={2: 9.0}),
patch(f"{_CONF}.fetch_hltb_confidence", side_effect=_DOWN),
patch(f"{_CONF}.save_hltb_cache") as save,
):
result = _hltb_confidence.fetch_hltb_confidence_cached([(1, "Celeste")])
assert result == {2: 9.0}
save.assert_not_called()
def test_detail_missing_reports_nothing_fetched_when_hltb_is_down() -> None:
"""The detail backfill returns 0 and writes nothing when HLTB is down."""
with (
patch(f"{_CONF}.fetch_hltb_times_timed", side_effect=_DOWN),
patch(f"{_CONF}.save_hltb_cache") as save,
):
assert _hltb_confidence.fetch_hltb_detail_missing([(1, "Celeste")]) == 0
save.assert_not_called()
def test_submit_url_is_none_when_hltb_is_down() -> None:
"""No URL instead of a crash when the lookup cannot run."""
with patch(f"{_CONF}.fetch_hltb_times", side_effect=_DOWN):
assert _hltb_confidence.get_hltb_submit_url("Celeste") is None
def test_refresh_keeps_cache_when_hltb_is_down() -> None:
"""A re-search that cannot run returns the loaded caches and saves nothing."""
with (
patch(f"{_CONF}.load_hltb_cache", return_value={1: 9.0}),
patch(f"{_CONF}.load_hltb_polls_cache", return_value={1: 30}),
patch(f"{_CONF}.load_hltb_count_comp_cache", return_value={1: 80}),
patch(f"{_CONF}.fetch_hltb_confidence", side_effect=_DOWN),
patch(f"{_CONF}.save_hltb_cache") as save,
):
result = _hltb_confidence.refresh_confidence([(1, "Celeste")])
assert result == ({1: 9.0}, {1: 30}, {1: 80})
save.assert_not_called()
def test_times_cached_keeps_cache_when_hltb_is_down() -> None:
"""The times fetch returns the cache as loaded and saves nothing."""
cached = "steam_backlog_enforcer._hltb_cached"
with (
patch(f"{cached}.load_hltb_cache", return_value={2: 9.0}),
patch(f"{cached}.fetch_hltb_times_timed", side_effect=_DOWN),
patch(f"{cached}.save_hltb_cache") as save,
):
result = _hltb_cached.fetch_hltb_times_cached([(1, "Celeste")])
assert result == {2: 9.0}
save.assert_not_called()

View File

@ -6,8 +6,11 @@ import asyncio
from typing import Self
from unittest.mock import patch
import pytest
from steam_backlog_enforcer._hltb_confidence import fetch_hltb_confidence_cached
from steam_backlog_enforcer._hltb_search import _AuthInfo
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
from steam_backlog_enforcer._hltb_types import _AuthInfo
from steam_backlog_enforcer.hltb import (
_fetch_batch_confidence_only,
fetch_hltb_confidence,
@ -39,11 +42,9 @@ class TestConfidenceHelpers:
patch(f"{PKG}.aiohttp.TCPConnector"),
patch(f"{PKG}._get_hltb_search_url", return_value="https://example"),
patch(f"{PKG}._get_auth_info", return_value=None),
pytest.raises(HLTBUnavailableError, match="nothing refreshed"),
):
result = asyncio.run(
_fetch_batch_confidence_only([(1, "Game")], {}, {}, None),
)
assert result == []
asyncio.run(_fetch_batch_confidence_only([(1, "Game")], {}, {}, None))
def test_fetch_batch_confidence_only_handles_empty_hp_and_default_counts(
self,

View File

@ -0,0 +1,131 @@
"""Detail-page failures are counted; ``stats`` prints the tally and fails loud."""
from __future__ import annotations
import asyncio
from typing import Self
from unittest.mock import MagicMock, patch
import aiohttp
import pytest
from steam_backlog_enforcer import _stats
from steam_backlog_enforcer._hltb_comp_extract import _fetch_detail_one
from steam_backlog_enforcer._hltb_failures import (
failure_summary_lines,
record_failure,
record_fatal,
record_success,
reset_failures,
)
@pytest.fixture(autouse=True)
def _fresh_tally() -> None:
reset_failures()
class _Page:
"""Fake detail-page response."""
def __init__(self, status: int) -> None:
"""Store the status."""
self.status = status
async def __aenter__(self) -> Self:
"""Enter."""
return self
async def __aexit__(self, *args: object) -> None:
"""Exit."""
async def text(self) -> str:
"""Empty page."""
return ""
def _detail(outcome: object) -> None:
session = MagicMock()
if isinstance(outcome, BaseException):
session.get.side_effect = outcome
else:
session.get.return_value = outcome
asyncio.run(_fetch_detail_one(asyncio.Semaphore(1), session, 77))
@pytest.mark.parametrize(
("outcome", "label"),
[
(_Page(429), "detail HTTP 429"),
(_Page(500), "detail HTTP 500"),
(TimeoutError(), "detail timeouts"),
(aiohttp.ClientConnectionError(), "detail connection errors"),
],
)
def test_detail_failure_is_counted(outcome: object, label: str) -> None:
"""A failed detail fetch shows up in the summary line."""
_detail(outcome)
assert f"{label} x1" in failure_summary_lines()[0]
def test_missing_detail_page_is_not_a_failure() -> None:
"""A 404 is HLTB's real answer, not a failed lookup."""
_detail(_Page(404))
assert failure_summary_lines() == []
def _stats_with(effect: object) -> None:
with patch.object(_stats, "_print_stats", side_effect=effect):
_stats.cmd_stats(MagicMock(), MagicMock())
def test_stats_ends_with_failure_line_and_exits_zero(
capsys: pytest.CaptureFixture[str],
) -> None:
"""Old data survived: report the failures, exit 0."""
_stats_with(lambda *_a: record_failure("HTTP 429"))
out = capsys.readouterr().out.rstrip().splitlines()
assert out[-1] == (
"HLTB: 1 lookups failed (HTTP 429 x1), cache kept for those games"
)
def test_stats_exits_nonzero_when_a_batch_refreshed_nothing(
capsys: pytest.CaptureFixture[str],
) -> None:
"""A total failure (auth init or every lookup) is a non-zero exit."""
msg = "HLTB: rate-limited, nothing refreshed (auth init: HTTP 429)"
with pytest.raises(SystemExit) as exc:
_stats_with(lambda *_a: record_fatal(msg))
assert exc.value.code == 1
assert capsys.readouterr().out.rstrip().splitlines()[-1] == msg
def test_stats_resets_the_tally_first(capsys: pytest.CaptureFixture[str]) -> None:
"""Failures from an earlier command in the same process are not re-reported."""
record_fatal("stale")
_stats_with(None)
assert "stale" not in capsys.readouterr().out
def test_stats_exits_zero_when_another_lookup_succeeded(
capsys: pytest.CaptureFixture[str],
) -> None:
"""One refreshed game anywhere in the command means old data survived: exit 0."""
msg = "HLTB: all 3 lookups failed, nothing refreshed"
def one_success_then_a_fatal_batch(*_a: object) -> None:
record_success()
record_fatal(msg)
_stats_with(one_success_then_a_fatal_batch)
assert capsys.readouterr().out.rstrip().splitlines()[-1] == msg
def test_repeated_fatal_lines_print_once() -> None:
"""Four batches failing auth the same way give one line, not four."""
for _ in range(4):
record_fatal("HLTB: rate-limited, nothing refreshed (auth init: timeout)")
assert failure_summary_lines() == [
"HLTB: rate-limited, nothing refreshed (auth init: timeout)"
]

View File

@ -0,0 +1,126 @@
"""HLTB requests are paced: one at a time, 1.1 s gap after each request (~55/min).
HLTB rate-limits a burst after ~60 lookups; a full refresh is ~800 games.
Fake transport only: no request ever leaves the process.
"""
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING, cast
from unittest.mock import patch
import pytest
from steam_backlog_enforcer import _hltb_pacing
from steam_backlog_enforcer._hltb_comp_extract import _fetch_detail_one
from steam_backlog_enforcer._hltb_failures import BREAKER_LIMIT, shared_breaker
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
from steam_backlog_enforcer._hltb_types import MAX_CONCURRENT, _AuthInfo
from steam_backlog_enforcer.tests._hltb_fake_transport import (
MATCH,
NO_MATCH,
FakeResp,
FakeSession,
)
if TYPE_CHECKING:
import aiohttp
_INTERVAL = 1.1
class _DetailSession:
"""Fake session answering every detail GET with a 404."""
def __init__(self) -> None:
self.gets = 0
def get(self, _url: str, *, headers: object) -> FakeResp:
del headers
self.gets += 1
return FakeResp(404)
def _ctx(session: FakeSession) -> _SearchCtx:
return _SearchCtx(
session=cast("aiohttp.ClientSession", session),
search_url="https://example.invalid/api/finder",
headers={},
cache={},
auth=_AuthInfo("t"),
counter={"done": 0, "found": 0, "failed": 0},
)
def _record_sleeps(sem: asyncio.Semaphore) -> tuple[list[float], object]:
"""Fake ``asyncio.sleep`` that records each pause and that a slot is held."""
pauses: list[float] = []
async def fake_sleep(seconds: float) -> None:
assert sem.locked(), "the pause must hold the request slot"
pauses.append(seconds)
return pauses, fake_sleep
def test_one_request_in_flight() -> None:
"""HLTB sees one request in flight at a time."""
assert MAX_CONCURRENT == 1
def test_default_interval_stays_under_the_limit() -> None:
"""1.1 s between requests keeps us at ~55/min, under the ~60/min limit."""
assert _hltb_pacing.REQUEST_INTERVAL_S == _INTERVAL
@pytest.mark.parametrize("body", [MATCH, NO_MATCH])
def test_each_search_post_is_followed_by_a_pause(body: dict[str, object]) -> None:
"""Every search POST, match or not, waits the interval inside its slot."""
session = FakeSession({"*": FakeResp(200, body)})
ctx = _ctx(session)
async def run() -> list[float]:
sem = asyncio.Semaphore(1)
pauses, fake_sleep = _record_sleeps(sem)
with patch.object(_hltb_pacing, "_sleep", fake_sleep):
await _search_one(sem, ctx, 1, "Vox Populi: Poland 2023")
return pauses
pauses = asyncio.run(run())
assert len(session.posts) >= 1
assert pauses == [_INTERVAL] * len(session.posts)
def test_skipped_lookup_after_breaker_does_not_pause() -> None:
"""Once the breaker has tripped nothing is sent, so nothing waits."""
session = FakeSession({"*": FakeResp(200, MATCH)})
ctx = _ctx(session)
ctx.breaker = shared_breaker()
for _ in range(BREAKER_LIMIT):
ctx.breaker.note(429)
async def run() -> list[float]:
sem = asyncio.Semaphore(1)
pauses, fake_sleep = _record_sleeps(sem)
with patch.object(_hltb_pacing, "_sleep", fake_sleep):
await _search_one(sem, ctx, 1, "Celeste")
return pauses
assert asyncio.run(run()) == []
assert session.posts == []
def test_each_detail_get_is_followed_by_a_pause() -> None:
"""Detail-page GETs are paced the same way as search POSTs."""
session = _DetailSession()
async def run() -> list[float]:
sem = asyncio.Semaphore(1)
pauses, fake_sleep = _record_sleeps(sem)
with patch.object(_hltb_pacing, "_sleep", fake_sleep):
await _fetch_detail_one(sem, cast("aiohttp.ClientSession", session), 77)
return pauses
assert asyncio.run(run()) == [_INTERVAL]
assert session.gets == 1

View File

@ -0,0 +1,75 @@
"""Refetch paths must not lose polls/count_comp when the refetch writes nothing.
The fetch itself is stubbed to a no-op, which is exactly what a failed HLTB
lookup now does: it leaves the in-memory caches untouched.
"""
from __future__ import annotations
import json
from unittest.mock import patch
from steam_backlog_enforcer import _hltb_confidence, _hltb_types, _scanning_confidence
from steam_backlog_enforcer.steam_api import GameInfo
_ENTRY = {
"hours": 20.0,
"polls": 100,
"count_comp": 500,
"rush_hours": 15.0,
"leisure_100h": 30.0,
"hltb_game_id": 77,
}
def _seed(entries: dict[int, dict[str, float]]) -> None:
_hltb_types.HLTB_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
_hltb_types.HLTB_CACHE_FILE.write_text(
json.dumps({str(k): v for k, v in entries.items()}), encoding="utf-8"
)
def _raw() -> dict[str, dict[str, float]]:
data: dict[str, dict[str, float]] = json.loads(
_hltb_types.HLTB_CACHE_FILE.read_text(encoding="utf-8")
)
return data
def test_refetch_poll_counts_keeps_polls_on_failure() -> None:
"""refetch_poll_counts used to drop the entry and restore only its hours."""
_seed({1: dict(_ENTRY)})
with patch.object(_hltb_confidence, "fetch_hltb_confidence"):
polls = _hltb_confidence.refetch_poll_counts([(1, "Game")])
assert polls[1] == 100
assert _raw()["1"] == _ENTRY
def test_confidence_refresh_keeps_entry_on_failure() -> None:
"""The candidate confidence refresh popped hours/polls/count_comp to disk."""
_seed({1: dict(_ENTRY)})
game = GameInfo(
app_id=1,
name="Game",
total_achievements=10,
unlocked_achievements=0,
playtime_minutes=0,
)
with patch.object(_hltb_confidence, "fetch_hltb_confidence"):
_scanning_confidence._refresh_candidate_confidence_batch([game], force=True)
assert _raw()["1"] == _ENTRY
assert (game.comp_100_count, game.count_comp) == (100, 500)
def test_detail_missing_keeps_known_miss_on_failure() -> None:
"""A cached miss (-1) must not vanish from the cache when the refetch fails."""
miss = dict(
_ENTRY, hours=-1.0, rush_hours=-1.0, leisure_100h=-1.0, polls=0, count_comp=0
)
_seed({1: miss, 2: dict(_ENTRY, rush_hours=-1.0)})
with patch.object(_hltb_confidence, "fetch_hltb_times_timed", return_value=0.0):
_hltb_confidence.fetch_hltb_detail_missing([(1, "Gone"), (2, "Known")])
raw = _raw()
assert raw["1"]["hours"] == -1
assert raw["2"]["hours"] == 20.0
assert raw["2"]["polls"] == 100

View File

@ -54,7 +54,7 @@ def _make_ctx(
search_url="https://example.com/search",
headers={},
cache=cache if cache is not None else {},
counter={"done": 0, "found": 0},
counter={"done": 0, "found": 0, "failed": 0},
total=1,
progress_cb=progress_cb,
)
@ -118,6 +118,8 @@ class TestSearchOne:
ctx = _make_ctx(session)
result = asyncio.run(_search_one(asyncio.Semaphore(1), ctx, 440, "TF2"))
assert result is None
assert 440 not in ctx.cache # a failed lookup is not a miss
assert ctx.counter["failed"] == 1
def test_non_200(self) -> None:
"""Test non 200."""
@ -125,6 +127,8 @@ class TestSearchOne:
ctx = _make_ctx(_make_session(resp))
result = asyncio.run(_search_one(asyncio.Semaphore(1), ctx, 440, "TF2"))
assert result is None
assert 440 not in ctx.cache # a failed lookup is not a miss
assert ctx.counter["failed"] == 1
def test_fallback_name_without_year_suffix(self) -> None:
"""Test fallback name without year suffix."""

View File

@ -0,0 +1,124 @@
"""Batch-level HLTB failure handling: auth init, total failure, circuit breaker."""
from __future__ import annotations
import asyncio
from unittest.mock import patch
import aiohttp
import pytest
from steam_backlog_enforcer import _hltb_search
from steam_backlog_enforcer._hltb_failures import (
HLTBUnavailableError,
failure_summary_lines,
reset_failures,
)
from steam_backlog_enforcer._hltb_search import _search_batch
from steam_backlog_enforcer._hltb_types import _HLTBExtras
from steam_backlog_enforcer.tests._hltb_fake_transport import (
MATCH,
FakeInitSession,
FakeResp,
)
@pytest.fixture(autouse=True)
def _fresh_tally() -> None:
reset_failures()
def _batch(
status: int,
routes: dict[str, object],
games: list[tuple[int, str]],
fake: FakeInitSession | None = None,
) -> FakeInitSession:
fake = fake or FakeInitSession(status, routes)
with (
patch.object(
_hltb_search, "_get_hltb_search_url", return_value="https://x.invalid/f"
),
patch.object(_hltb_search.aiohttp, "ClientSession", lambda **_k: fake),
patch.object(_hltb_search.aiohttp, "TCPConnector", lambda **_k: None),
):
asyncio.run(_search_batch(games, {}, {}, None, _HLTBExtras()))
return fake
@pytest.mark.parametrize(
"init", [429, aiohttp.ClientConnectionError("no network")], ids=["429", "offline"]
)
def test_auth_init_failure_fails_the_batch_up_front(
init: int | BaseException,
) -> None:
"""A failed /init means nothing can refresh: raise instead of returning []."""
with pytest.raises(HLTBUnavailableError, match="rate-limited, nothing refreshed"):
_batch(init, {"*": FakeResp(200, MATCH)}, [(1, "Vox Populi")])
assert any("nothing refreshed" in ln for ln in failure_summary_lines())
def test_empty_batch_is_not_a_failure() -> None:
"""No games: no request, no error."""
fake = _batch(429, {}, [])
assert fake.search.posts == []
assert failure_summary_lines() == []
def test_breaker_trips_after_five_consecutive_429s() -> None:
"""After 5 straight 429s the rest of the batch sends no request at all."""
fake = FakeInitSession(200, {"*": FakeResp(429)})
games = [(aid, "Celeste") for aid in range(10, 18)]
with (
patch.object(
_hltb_search, "_get_hltb_search_url", return_value="https://x.invalid/f"
),
patch.object(_hltb_search.aiohttp, "ClientSession", lambda **_k: fake),
patch.object(_hltb_search.aiohttp, "TCPConnector", lambda **_k: None),
pytest.raises(HLTBUnavailableError, match="all 8 lookups failed"),
):
asyncio.run(_search_batch(games, {}, {}, None, _HLTBExtras()))
assert len(fake.search.posts) == 5
lines = failure_summary_lines()
assert lines[0] == (
"HLTB: 8 lookups failed (HTTP 429 x5, skipped after rate-limit x3),"
" cache kept for those games"
)
assert "HLTB: rate-limited, 8 games not refreshed" in lines
def test_batch_where_every_lookup_failed_raises() -> None:
"""Every lookup failing is a total failure."""
with pytest.raises(HLTBUnavailableError, match="all 2 lookups failed"):
_batch(200, {"*": FakeResp(500)}, [(1, "Celeste"), (2, "Celeste")])
def test_batch_with_one_success_does_not_raise() -> None:
"""At least one success: the batch completes and failures are only reported."""
_batch(
200,
{"Vox Populi": FakeResp(200, MATCH), "*": FakeResp(500)},
[(1, "Vox Populi"), (2, "Celeste")],
)
assert failure_summary_lines()[0].startswith("HLTB: 1 lookups failed (HTTP 500 x1)")
def test_tripped_breaker_skips_later_batches_of_the_same_command() -> None:
"""Once HLTB rate-limits us, later batches send no request (not even /init)."""
with pytest.raises(HLTBUnavailableError):
_batch(200, {"*": FakeResp(429)}, [(aid, "Celeste") for aid in range(5)])
fake = FakeInitSession(200, {"*": FakeResp(200, MATCH)})
with pytest.raises(HLTBUnavailableError, match="rate-limited"):
_batch(200, {}, [(1, "Vox"), (2, "Vox")], fake)
assert (fake.gets, fake.search.posts) == (0, [])
# Games 1 and 2 were in both batches: count games, not lookups.
assert "HLTB: rate-limited, 5 games not refreshed" in failure_summary_lines()
def test_reset_clears_a_tripped_breaker() -> None:
"""A new command starts with HLTB assumed reachable again."""
with pytest.raises(HLTBUnavailableError):
_batch(200, {"*": FakeResp(429)}, [(aid, "Celeste") for aid in range(5)])
reset_failures()
fake = _batch(200, {"*": FakeResp(200, MATCH)}, [(1, "Vox Populi")])
assert len(fake.search.posts) == 1

View File

@ -0,0 +1,124 @@
"""A failed HLTB search must keep the cached entry; only a clean no-match is a miss.
Fake transport only: no request ever leaves the process.
"""
from __future__ import annotations
import asyncio
from typing import cast
import aiohttp
import pytest
from steam_backlog_enforcer._hltb_failures import (
failure_summary_lines,
reset_failures,
)
from steam_backlog_enforcer._hltb_search_one import _search_one, _SearchCtx
from steam_backlog_enforcer._hltb_types import _AuthInfo, _HLTBExtras
from steam_backlog_enforcer.tests._hltb_fake_transport import (
MATCH,
NO_MATCH,
FakeResp,
FakeSession,
)
def _ctx(session: FakeSession, total: int = 1) -> _SearchCtx:
return _SearchCtx(
session=cast("aiohttp.ClientSession", session),
search_url="https://example.invalid/api/finder",
headers={},
cache={1: 50.0},
polls={1: 120},
extras=_HLTBExtras(count_comp={1: 900}),
auth=_AuthInfo("t"),
counter={"done": 0, "found": 0, "failed": 0},
total=total,
)
def _run(ctx: _SearchCtx, app_id: int, name: str) -> None:
asyncio.run(_search_one(asyncio.Semaphore(1), ctx, app_id, name))
@pytest.fixture(autouse=True)
def _fresh_tally() -> None:
reset_failures()
_FAILURES = {
"429": FakeResp(429),
"500": FakeResp(500),
"403": FakeResp(403),
"timeout": TimeoutError(),
"client_error": aiohttp.ClientConnectionError("refused"),
"bad_json": FakeResp(200, None),
}
@pytest.mark.parametrize("kind", sorted(_FAILURES))
def test_failed_search_keeps_prior_entry(kind: str) -> None:
"""Any non-clean answer leaves hours, polls and count_comp untouched."""
ctx = _ctx(FakeSession({"*": _FAILURES[kind]}))
_run(ctx, 1, "Celeste")
assert ctx.cache[1] == 50.0
assert ctx.polls[1] == 120
assert ctx.extras.count_comp[1] == 900
assert ctx.counter["failed"] == 1
assert ctx.counter["found"] == 0
def test_failed_variant_blocks_a_miss() -> None:
"""One variant no-match + one variant 429 is a failure, not a miss."""
routes = {"Vox Populi Poland 2023": FakeResp(200, NO_MATCH), "*": FakeResp(429)}
ctx = _ctx(FakeSession(routes))
_run(ctx, 1, "Vox Populi: Poland 2023")
assert (ctx.cache[1], ctx.polls[1], ctx.extras.count_comp[1]) == (50.0, 120, 900)
assert ctx.counter["failed"] == 1
def test_match_beats_failed_sibling_variant() -> None:
"""A real match replaces the entry even when another variant failed."""
routes = {"Vox Populi Poland 2023": FakeResp(429), "*": FakeResp(200, MATCH)}
ctx = _ctx(FakeSession(routes))
_run(ctx, 1, "Vox Populi: Poland 2023")
assert (ctx.cache[1], ctx.polls[1], ctx.extras.count_comp[1]) == (10.0, 42, 300)
assert ctx.counter == {"done": 1, "found": 1, "failed": 0}
def test_clean_no_match_still_writes_a_miss() -> None:
"""Every variant answered 200 with nothing usable: that is a real miss."""
ctx = _ctx(FakeSession({"*": FakeResp(200, NO_MATCH)}))
_run(ctx, 1, "Vox Populi: Poland 2023")
assert (ctx.cache[1], ctx.polls[1], ctx.extras.count_comp[1]) == (-1, 0, 0)
assert ctx.counter["failed"] == 0
def test_failure_without_prior_entry_writes_nothing() -> None:
"""An uncached game that fails stays uncached so the next run retries it."""
ctx = _ctx(FakeSession({"*": FakeResp(429)}))
_run(ctx, 2, "Celeste")
assert 2 not in ctx.cache
assert 2 not in ctx.polls
assert 2 not in ctx.extras.count_comp
assert ctx.counter == {"done": 1, "found": 0, "failed": 1}
def test_summary_line_counts_reasons() -> None:
"""The end-of-run line names every failure reason with its count."""
ctx = _ctx(FakeSession({"*": FakeResp(429)}), total=2)
_run(ctx, 1, "Celeste")
ctx.session = cast("aiohttp.ClientSession", FakeSession({"*": TimeoutError()}))
_run(ctx, 1, "Celeste")
assert failure_summary_lines()[0] == (
"HLTB: 2 lookups failed (HTTP 429 x1, timeouts x1), cache kept for those games"
)
def test_no_failures_no_summary() -> None:
"""A clean run prints no failure line."""
ctx = _ctx(FakeSession({"*": FakeResp(200, MATCH)}))
_run(ctx, 1, "Vox Populi")
assert failure_summary_lines() == []

View File

@ -6,6 +6,9 @@ import asyncio
from typing import TYPE_CHECKING, Any, Self
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from steam_backlog_enforcer._hltb_failures import HLTBUnavailableError
from steam_backlog_enforcer._hltb_search import (
_fetch_batch,
_SearchCtx,
@ -73,9 +76,9 @@ class TestFetchBatchHltb:
new_callable=AsyncMock,
return_value=None,
),
pytest.raises(HLTBUnavailableError, match="nothing refreshed"),
):
results = asyncio.run(_fetch_batch([(440, "TF2")], {}, {}, None))
assert results == []
asyncio.run(_fetch_batch([(440, "TF2")], {}, {}, None))
class TestPickBestEntry:

View File

@ -147,7 +147,9 @@ class TestSearchOneGroup2:
# Set done to one less than _SAVE_INTERVAL so it triggers save
ctx.counter["done"] = _SAVE_INTERVAL - 1
with patch("steam_backlog_enforcer._hltb_search.save_hltb_cache") as mock_save:
with patch(
"steam_backlog_enforcer._hltb_search_one.save_hltb_cache"
) as mock_save:
asyncio.run(_search_one(asyncio.Semaphore(1), ctx, 440, "TF2"))
mock_save.assert_called_once()

View File

@ -57,19 +57,26 @@ class TestBackfillPollsForFinished:
json.dumps({"1": {"hours": 2.0, "polls": 0}}), encoding="utf-8"
)
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
"""Test fake fetch."""
data = json.loads(cache_file.read_text(encoding="utf-8"))
def fake_fetch(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Test fake search: write into the caches it is handed."""
for aid, _name in games:
data[str(aid)] = {"hours": 2.0, "polls": 9}
cache_file.write_text(json.dumps(data), encoding="utf-8")
return {aid: 2.0 for aid, _ in games}
cache[aid], polls[aid], count_comp[aid] = 2.0, 9, 0
return []
with (
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
patch(f"{_CMD}.load_snapshot", return_value=[{"app_id": 1, "name": "G"}]),
patch(f"{_CONF}.fetch_hltb_confidence_cached", side_effect=fake_fetch),
patch(
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
side_effect=fake_fetch,
),
patch(f"{_CMD}._echo"),
):
result = _cmd_done._backfill_polls_for_finished(_state([1]))
@ -82,19 +89,26 @@ class TestBackfillPollsForFinished:
json.dumps({"7": {"hours": 1.0, "polls": 0}}), encoding="utf-8"
)
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
"""Test fake fetch."""
data = json.loads(cache_file.read_text(encoding="utf-8"))
def fake_fetch(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Test fake search: write into the caches it is handed."""
for aid, _name in games:
data[str(aid)] = {"hours": 1.0, "polls": 4}
cache_file.write_text(json.dumps(data), encoding="utf-8")
return {aid: 1.0 for aid, _ in games}
cache[aid], polls[aid], count_comp[aid] = 1.0, 4, 0
return []
with (
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
patch(f"{_CMD}.load_snapshot", return_value=[{"app_id": 7, "name": "G"}]),
patch(f"{_CONF}.fetch_hltb_confidence_cached", side_effect=fake_fetch),
patch(
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
side_effect=fake_fetch,
),
patch(f"{_CMD}._echo"),
):
result = _cmd_done._backfill_polls_for_finished(
@ -109,20 +123,26 @@ class TestBackfillPollsForFinished:
json.dumps({"3": {"hours": 4.0, "polls": 0}}), encoding="utf-8"
)
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
# Simulate a refetch returning a miss (hours -1, polls 0).
"""Test fake fetch."""
data = json.loads(cache_file.read_text(encoding="utf-8"))
def fake_fetch(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Test fake search: a clean miss (hours -1, polls 0)."""
for aid, _name in games:
data[str(aid)] = {"hours": -1, "polls": 0}
cache_file.write_text(json.dumps(data), encoding="utf-8")
return {aid: -1 for aid, _ in games}
cache[aid], polls[aid], count_comp[aid] = -1, 0, 0
return []
with (
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
patch(f"{_CMD}.load_snapshot", return_value=[{"app_id": 3, "name": "G"}]),
patch(f"{_CONF}.fetch_hltb_confidence_cached", side_effect=fake_fetch),
patch(
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
side_effect=fake_fetch,
),
patch(f"{_CMD}._echo"),
):
_cmd_done._backfill_polls_for_finished(_state([3]))

View File

@ -149,19 +149,23 @@ class TestScanningPollsIntegration:
json.dumps({"2": {"hours": 3.0, "polls": 0}}), encoding="utf-8"
)
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
"""Test fake fetch."""
data = json.loads(cache_file.read_text(encoding="utf-8"))
def fake_fetch(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Test fake search: write into the caches it is handed."""
for aid, _name in games:
data[str(aid)] = {"hours": 3.0, "polls": 8}
cache_file.write_text(json.dumps(data), encoding="utf-8")
return {aid: 3.0 for aid, _ in games}
cache[aid], polls[aid], count_comp[aid] = 3.0, 8, 0
return []
with (
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
patch(
f"{_CONF}.fetch_hltb_confidence_cached",
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
side_effect=fake_fetch,
),
):
@ -186,19 +190,23 @@ class TestScanningPollsIntegration:
json.dumps({"2": {"hours": 9.0, "polls": 0}}), encoding="utf-8"
)
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
"""Test fake fetch."""
data = json.loads(cache_file.read_text(encoding="utf-8"))
def fake_fetch(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Test fake search: write into the caches it is handed."""
for aid, _name in games:
data[str(aid)] = {"hours": -1, "polls": 0}
cache_file.write_text(json.dumps(data), encoding="utf-8")
return {aid: -1 for aid, _ in games}
cache[aid], polls[aid], count_comp[aid] = -1, 0, 0
return []
with (
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
patch(
f"{_CONF}.fetch_hltb_confidence_cached",
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
side_effect=fake_fetch,
),
):

View File

@ -76,7 +76,9 @@ class TestScanningPollsIntegrationGroup2Group2:
comp_100_count=3,
count_comp=15,
)
with patch(f"{_SCANCONF}.fetch_hltb_confidence_cached") as mock_fetch:
with patch(
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence"
) as mock_fetch:
_scanning_confidence._refresh_candidate_confidence(game)
mock_fetch.assert_not_called()
@ -99,18 +101,23 @@ class TestScanningPollsIntegrationGroup2Group2:
count_comp=0,
)
def fake_fetch(_games: list[tuple[int, str]]) -> dict[int, float]:
"""Test fake fetch."""
data = json.loads(cache_file.read_text(encoding="utf-8"))
data["1"] = {"hours": 4.0, "polls": 3, "count_comp": 15}
cache_file.write_text(json.dumps(data), encoding="utf-8")
return {1: 4.0}
def fake_fetch(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Test fake search: write into the caches it is handed."""
del games
cache[1], polls[1], count_comp[1] = 4.0, 3, 15
return []
with (
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
patch(
f"{_SCANCONF}.fetch_hltb_confidence_cached",
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
side_effect=fake_fetch,
),
patch(f"{_SCANCONF}._echo"),
@ -151,20 +158,24 @@ class TestScanningPollsIntegrationGroup2Group2:
count_comp=0,
)
def fake_fetch(games: list[tuple[int, str]]) -> dict[int, float]:
"""Test fake fetch."""
def fake_fetch(
games: list[tuple[int, str]],
cache: dict[int, float],
polls: dict[int, int],
count_comp: dict[int, int],
**_kw: object,
) -> list[object]:
"""Test fake search: write into the caches it is handed."""
assert sorted(games) == [(1, "A"), (2, "B")]
data = json.loads(cache_file.read_text(encoding="utf-8"))
data["1"] = {"hours": 4.0, "polls": 3, "count_comp": 15}
data["2"] = {"hours": 5.0, "polls": 3, "count_comp": 15}
cache_file.write_text(json.dumps(data), encoding="utf-8")
return {1: 4.0, 2: 5.0}
cache[1], polls[1], count_comp[1] = 4.0, 3, 15
cache[2], polls[2], count_comp[2] = 5.0, 3, 15
return []
with (
patch(f"{_TYPES}.HLTB_CACHE_FILE", cache_file),
patch(f"{_TYPES}.CONFIG_DIR", tmp_path),
patch(
f"{_SCANCONF}.fetch_hltb_confidence_cached",
"steam_backlog_enforcer._hltb_confidence.fetch_hltb_confidence",
side_effect=fake_fetch,
) as mock_fetch,
patch(f"{_SCANCONF}._echo"),

View File

@ -52,7 +52,7 @@ class TestConfidenceHelpers:
game.comp_100_count = 3
game.count_comp = 15
with patch(
"steam_backlog_enforcer._scanning_confidence.fetch_hltb_confidence_cached",
"steam_backlog_enforcer._scanning_confidence.refresh_confidence",
) as mock_fetch:
_refresh_candidate_confidence_batch([game], force=False)
mock_fetch.assert_not_called()
@ -62,32 +62,17 @@ class TestConfidenceHelpers:
game = _game(app_id=30, name="C", hours=9.5)
game.comp_100_count = 0
game.count_comp = 0
with (
patch(
"steam_backlog_enforcer._scanning_confidence.load_hltb_cache",
side_effect=[{30: 9.5}, {30: -1.0}],
),
patch(
"steam_backlog_enforcer._scanning_confidence.load_hltb_polls_cache",
return_value={30: 0},
),
patch(
"steam_backlog_enforcer._scanning_confidence.load_hltb_count_comp_cache",
return_value={30: 0},
),
patch(
"steam_backlog_enforcer._scanning_confidence.fetch_hltb_confidence_cached",
return_value={30: -1.0},
),
patch(
"steam_backlog_enforcer._scanning_confidence.save_hltb_cache",
) as mock_save,
):
# refresh_confidence owns the hours-preservation rule (see
# test_hltb_refetch_failure_keeps_cache); this layer only applies polls.
with patch(
"steam_backlog_enforcer._scanning_confidence.refresh_confidence",
return_value=({30: 9.5}, {30: 4}, {30: 7}),
) as mock_refresh:
_refresh_candidate_confidence_batch([game], force=True)
mock_refresh.assert_called_once_with([(30, "C")])
assert game.completionist_hours == 9.5
saved_cache = mock_save.call_args.args[0]
assert saved_cache[30] == 9.5
assert (game.comp_100_count, game.count_comp) == (4, 7)
def test_filter_hltb_confident_candidates_skips_low_confidence(self) -> None:
"""Test filter hltb confident candidates skips low confidence."""

730
web/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -14,13 +14,13 @@
"coverage": "vitest run --coverage"
},
"dependencies": {
"@tanstack/react-query": "5.105.1",
"@tanstack/react-query": "5.107.1",
"@tanstack/react-router": "1.170.46",
"react": "19.3.1",
"react-dom": "19.3.1"
},
"devDependencies": {
"@babel/core": "8.0.10",
"@babel/core": "8.1.0",
"@eslint/js": "10.0.1",
"@rolldown/plugin-babel": "0.3.0",
"@testing-library/dom": "10.4.2",
@ -30,17 +30,17 @@
"@types/node": "26.6.6",
"@types/react": "19.3.0",
"@types/react-dom": "19.3.0",
"@vitejs/plugin-react": "6.1.2",
"@vitest/coverage-v8": "5.0.5",
"@vitejs/plugin-react": "6.2.1",
"@vitest/coverage-v8": "5.1.0",
"babel-plugin-react-compiler": "1.0.0",
"eslint": "10.14.0",
"eslint-plugin-react-hooks": "7.1.1",
"eslint-plugin-react-refresh": "0.5.7",
"globals": "17.14.0",
"jsdom": "30.1.3",
"jsdom": "31.0.0",
"typescript": "6.0.3",
"typescript-eslint": "8.72.1",
"vite": "8.3.5",
"vitest": "5.0.5"
"vite": "8.3.6",
"vitest": "5.1.0"
}
}

File diff suppressed because it is too large Load Diff