"""Internal HLTB search helpers: URL discovery, auth, matching, and batch fetch.""" from __future__ import annotations import asyncio from dataclasses import dataclass, field from http import HTTPStatus import logging import aiohttp from steam_backlog_enforcer._hltb_detail import ( _fetch_leisure_times, ) from steam_backlog_enforcer._hltb_matching import ( _build_result_from_best, _build_search_variants, ) from steam_backlog_enforcer._hltb_search_api import ( _build_search_payload, _collect_candidates, _get_auth_info, _get_hltb_search_url, _pick_best_hltb_entry, forget_hltb_search_url, ) from steam_backlog_enforcer._hltb_types import ( _SAVE_INTERVAL, MAX_CONCURRENT, HLTBResult, ProgressCb, _AuthInfo, _HLTBExtras, save_hltb_cache, ) logger = logging.getLogger(__name__) # When extended entry has ≥ this many times more hours than the exact match, # prefer it even if its confidence count is lower. _EXTENDED_DOMINANCE_RATIO = 4.0 # Minimum combined confidence for the dominance path (avoids picking entries # that have almost no data at all). _EXTENDED_MIN_CONFIDENCE = 3 # ────────────────────────────────────────────────────────────── # HLTB API setup (done once, not per-request like the library) # ────────────────────────────────────────────────────────────── # ────────────────────────────────────────────────────────────── # Async fetching with shared session & progress # ────────────────────────────────────────────────────────────── @dataclass class _SearchCtx: """Shared context for HLTB search requests.""" session: aiohttp.ClientSession search_url: str headers: dict[str, str] cache: dict[int, float] polls: dict[int, int] = field(default_factory=dict) extras: _HLTBExtras = field(default_factory=_HLTBExtras) auth: _AuthInfo | None = None counter: dict[str, int] = field(default_factory=dict) total: int = 0 progress_cb: ProgressCb | None = None async def _search_one( sem: asyncio.Semaphore, ctx: _SearchCtx, app_id: int, name: str, ) -> HLTBResult | None: """Search HLTB for one game via direct POST, update cache.""" async with sem: result: HLTBResult | None = None for query_name in _build_search_variants(name): payload = _build_search_payload(query_name, ctx.auth) try: async with ctx.session.post( ctx.search_url, headers=ctx.headers, data=payload, ) as resp: if resp.status != HTTPStatus.OK: continue data = await resp.json() candidates = _collect_candidates(query_name, data) # When we stripped ": subtitle" from the original name to # get query_name, only keep full-edition entries (those # whose HLTB name starts with query_name + ":"/"-") or # exact name/alias matches. This prevents "Vox Populi" # (stripped from "Vox Populi: Poland 2023") from falsely # matching "Vox Populi Vox Dei 2". if ":" in name and ":" not in query_name: lower_q = query_name.lower() candidates = [ (e, s) for e, s in candidates if (e.get("game_name") or "").lower() == lower_q or (e.get("game_alias") or "").lower() == lower_q or (e.get("game_name") or "") .lower() .startswith(lower_q + ":") or (e.get("game_name") or "") .lower() .startswith(lower_q + " -") ] best = _pick_best_hltb_entry(query_name, candidates) if best is None: continue result = _build_result_from_best(app_id, name, query_name, best) break except (TimeoutError, aiohttp.ClientError) as exc: logger.debug("HLTB search failed for '%s': %s", query_name, exc) # Update cache immediately (miss = -1). if result is not None: ctx.cache[app_id] = result.completionist_hours ctx.polls[app_id] = result.comp_100_count ctx.extras.count_comp[app_id] = result.count_comp if result.hltb_game_id > 0: ctx.extras.hltb_game_id[app_id] = result.hltb_game_id ctx.counter["found"] += 1 else: ctx.cache[app_id] = -1 ctx.polls[app_id] = 0 ctx.extras.count_comp[app_id] = 0 ctx.counter["done"] += 1 done = ctx.counter["done"] # Incremental save every _SAVE_INTERVAL lookups. if not done % _SAVE_INTERVAL: save_hltb_cache(ctx.cache, ctx.polls, ctx.extras) # Report progress. if ctx.progress_cb is not None: ctx.progress_cb(done, ctx.total, ctx.counter["found"], name) return result async def _search_batch( games: list[tuple[int, str]], cache: dict[int, float], polls: dict[int, int], progress_cb: ProgressCb | None, extras: _HLTBExtras, ) -> list[HLTBResult]: """Run the search-level HLTB lookups for a batch through one session. Hours, poll counts and the extras come from the search results alone; :func:`_fetch_batch` layers the per-game detail pages on top of this. """ # 1. Discover the search URL (sync, one-time). search_url = _get_hltb_search_url() logger.info("HLTB search URL: %s", search_url) timeout = aiohttp.ClientTimeout(total=20, sock_read=15) # 2. Get auth info (separate session — avoids reuse issues). async with aiohttp.ClientSession(timeout=timeout) as init_session: auth = await _get_auth_info(search_url, init_session) if auth is None: logger.warning("Could not get HLTB auth info, aborting fetch.") forget_hltb_search_url() return [] logger.info("HLTB auth token acquired.") # 3. Build shared headers for all search requests. headers: dict[str, str] = { "content-type": "application/json", "accept": "*/*", "User-Agent": ( "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0" ), "referer": "https://howlongtobeat.com/", "x-auth-token": auth.token, } if auth.hp_key: headers["x-hp-key"] = auth.hp_key headers["x-hp-val"] = auth.hp_val # 4. Fire all searches through a single persistent session. sem = asyncio.Semaphore(MAX_CONCURRENT) connector = aiohttp.TCPConnector( limit=MAX_CONCURRENT, keepalive_timeout=30, ) async with aiohttp.ClientSession( timeout=timeout, connector=connector, ) as session: ctx = _SearchCtx( session=session, search_url=search_url, headers=headers, cache=cache, polls=polls, extras=extras, auth=auth, counter={"done": 0, "found": 0}, total=len(games), progress_cb=progress_cb, ) tasks = [_search_one(sem, ctx, app_id, name) for app_id, name in games] results = await asyncio.gather(*tasks) return [r for r in results if r is not None] async def _fetch_batch( games: list[tuple[int, str]], cache: dict[int, float], polls: dict[int, int], progress_cb: ProgressCb | None, extras: _HLTBExtras | None = None, ) -> list[HLTBResult]: """Fetch HLTB data for a batch of games using one shared session.""" if extras is None: extras = _HLTBExtras() search_results = await _search_batch(games, cache, polls, progress_cb, extras) # 5. Fetch leisure times + DLC from game detail pages. logger.info( "Fetching leisure times for %d games from detail pages...", len(search_results), ) await _fetch_leisure_times( search_results, cache, polls, progress_cb=None, extras=extras, ) return search_results