From 72390acc2c0ba69cade0157b400c6211fbc66f2a Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Thu, 23 Jul 2026 12:50:26 +0200 Subject: [PATCH 01/24] Initial implementation: fast Steam installs via depotdownloader + steamcmd Wraps depotdownloader for the bulk content pull (bypasses Steam's own capped download speed) and steamcmd's app_update --validate to make Steam recognize the result, with a manual-manifest fallback for the confirmed steamcmd install-script segfault bug when no fix is available. Name search via store.steampowered.com is best-effort since that endpoint TLS-rejects scripted clients on this network; --appid plus steamcmd's own app_info_print is the reliable primary path. Co-Authored-By: Claude Sonnet 5 --- _installer_import/.gitignore | 5 + _installer_import/install.sh | 24 +++ _installer_import/pyproject.toml | 16 ++ _installer_import/steam_installer/__init__.py | 0 _installer_import/steam_installer/appinfo.py | 61 ++++++++ _installer_import/steam_installer/cli.py | 121 ++++++++++++++++ _installer_import/steam_installer/depotdl.py | 46 ++++++ _installer_import/steam_installer/library.py | 76 ++++++++++ .../steam_installer/recognize.py | 137 ++++++++++++++++++ _installer_import/steam_installer/resolve.py | 61 ++++++++ _installer_import/steam_installer/vdf.py | 120 +++++++++++++++ _installer_import/tests/test_library.py | 70 +++++++++ _installer_import/tests/test_recognize.py | 81 +++++++++++ _installer_import/tests/test_vdf.py | 68 +++++++++ 14 files changed, 886 insertions(+) create mode 100644 _installer_import/.gitignore create mode 100755 _installer_import/install.sh create mode 100644 _installer_import/pyproject.toml create mode 100644 _installer_import/steam_installer/__init__.py create mode 100644 _installer_import/steam_installer/appinfo.py create mode 100644 _installer_import/steam_installer/cli.py create mode 100644 _installer_import/steam_installer/depotdl.py create mode 100644 _installer_import/steam_installer/library.py create mode 100644 _installer_import/steam_installer/recognize.py create mode 100644 _installer_import/steam_installer/resolve.py create mode 100644 _installer_import/steam_installer/vdf.py create mode 100644 _installer_import/tests/test_library.py create mode 100644 _installer_import/tests/test_recognize.py create mode 100644 _installer_import/tests/test_vdf.py diff --git a/_installer_import/.gitignore b/_installer_import/.gitignore new file mode 100644 index 0000000..76180b2 --- /dev/null +++ b/_installer_import/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +*.egg-info/ diff --git a/_installer_import/install.sh b/_installer_import/install.sh new file mode 100755 index 0000000..54cd179 --- /dev/null +++ b/_installer_import/install.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Checks for depotdownloader and steamcmd on PATH. steamcmd is AUR-only and +# needs an interactive license/sudo prompt, so it's never installed for you -- +# this just tells you the command to run. + +set -euo pipefail + +missing=0 + +if ! command -v depotdownloader >/dev/null 2>&1; then + echo "missing: depotdownloader (see ~/depotdownloader)" >&2 + missing=1 +fi + +if ! command -v steamcmd >/dev/null 2>&1; then + echo "missing: steamcmd -- run: yay -S steamcmd" >&2 + missing=1 +fi + +if [[ "$missing" -eq 0 ]]; then + echo "All dependencies present." +fi + +exit "$missing" diff --git a/_installer_import/pyproject.toml b/_installer_import/pyproject.toml new file mode 100644 index 0000000..6c9315a --- /dev/null +++ b/_installer_import/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "steam-game-installer" +version = "0.1.0" +description = "Fast Steam game installs: depotdownloader for speed, steamcmd for recognition." +requires-python = ">=3.11" +dependencies = [] + +[project.scripts] +steam-game-installer = "steam_installer.cli:main" + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/_installer_import/steam_installer/__init__.py b/_installer_import/steam_installer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/_installer_import/steam_installer/appinfo.py b/_installer_import/steam_installer/appinfo.py new file mode 100644 index 0000000..9a9a081 --- /dev/null +++ b/_installer_import/steam_installer/appinfo.py @@ -0,0 +1,61 @@ +"""Look up app metadata via steamcmd's app_info_print (anonymous, no login needed).""" + +from __future__ import annotations + +import re +import subprocess +from dataclasses import dataclass + +from steam_installer import vdf + + +class AppInfoError(Exception): + pass + + +@dataclass +class AppMetadata: + appid: int + name: str + installdir: str + + +def _fetch_app_block(appid: int) -> dict: + result = subprocess.run( + ["steamcmd", "+login", "anonymous", "+app_info_print", str(appid), "+quit"], + capture_output=True, + text=True, + timeout=60, + ) + output = result.stdout + # app_info_print emits `""\n{ ... }` inside a stream of steamcmd's own log lines; + # slice from the first line that starts the VDF block onward. + match = re.search(rf'^"{appid}"\s*$', output, re.MULTILINE) + if not match: + raise AppInfoError(f"steamcmd did not return app info for {appid}:\n{output}") + data = vdf.parse(output[match.start() :]) + app_block = data.get(str(appid), {}) + if not isinstance(app_block, dict): + raise AppInfoError(f"unexpected app_info_print structure for {appid}") + return app_block + + +def get_metadata(appid: int) -> AppMetadata: + """Fetch the fields this project actually needs: display name and installdir.""" + app_block = _fetch_app_block(appid) + common = app_block.get("common", {}) + if not isinstance(common, dict): + raise AppInfoError(f"no 'common' section in app info for {appid}") + + name = common.get("name") + if not isinstance(name, str): + raise AppInfoError(f"no name field found for app {appid}") + + installdir = common.get("installdir") + if not isinstance(installdir, str): + config = app_block.get("config", {}) + installdir = config.get("installdir") if isinstance(config, dict) else None + if not isinstance(installdir, str): + raise AppInfoError(f"no installdir field found for app {appid}") + + return AppMetadata(appid=appid, name=name, installdir=installdir) diff --git a/_installer_import/steam_installer/cli.py b/_installer_import/steam_installer/cli.py new file mode 100644 index 0000000..c32e257 --- /dev/null +++ b/_installer_import/steam_installer/cli.py @@ -0,0 +1,121 @@ +"""Entry point: resolve -> confirm -> fast download -> make Steam recognize it.""" + +from __future__ import annotations + +import argparse +import shutil +import sys +import time + +from steam_installer import appinfo, depotdl, recognize, resolve +from steam_installer.library import find_steam_root, list_libraries, pick_install_target + + +def _require_tool(name: str, install_hint: str) -> None: + if shutil.which(name) is None: + print(f"error: `{name}` not found on PATH. {install_hint}", file=sys.stderr) + sys.exit(1) + + +def _resolve_appid(args: argparse.Namespace) -> int: + """Prefer an explicit --appid; fall back to search, which is best-effort + (store.steampowered.com is known to reject scripted clients on some + networks -- see resolve.py). Either way, the final confirmation uses the + real name from steamcmd's own app info, not whatever the search returned. + """ + if args.appid is not None: + return args.appid + + try: + results = resolve.search(args.name) + except resolve.SearchUnavailable as exc: + print(f"Store search unavailable ({exc}).", file=sys.stderr) + print( + "Find the numeric AppID yourself (e.g. from the store URL) and pass --appid.", + file=sys.stderr, + ) + sys.exit(1) + + chosen = resolve.prompt_confirm(results) + return chosen.appid + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Download an owned Steam game fast, then make Steam recognize it." + ) + parser.add_argument("name", nargs="?", default=None, help="Game name to search for") + parser.add_argument( + "--appid", type=int, default=None, help="Steam AppID (skips search)" + ) + parser.add_argument("--username", required=True, help="Steam account username") + parser.add_argument( + "--language", default=None, help="Language depot to fetch (default: prompts)" + ) + parser.add_argument( + "--max-downloads", type=int, default=depotdl.DEFAULT_MAX_DOWNLOADS + ) + args = parser.parse_args(argv) + + if args.appid is None and args.name is None: + parser.error("provide a game name to search for, or --appid directly") + + _require_tool("depotdownloader", "See ~/depotdownloader.") + _require_tool("steamcmd", "Run: yay -S steamcmd") + + appid = _resolve_appid(args) + metadata = appinfo.get_metadata(appid) + print(f"Confirmed: {metadata.name} (appid {metadata.appid})") + answer = input("Proceed? [y/N]: ").strip().lower() + if answer != "y": + print("Cancelled.") + return 1 + + steam_root = find_steam_root() + libraries = list_libraries(steam_root) + target = pick_install_target(libraries, metadata.installdir) + print(f"Install target: {target}") + + language = args.language + if language is None: + language = input("Language [english]: ").strip() or "english" + + print("Downloading via depotdownloader (fast path)...") + depotdl.download( + appid=metadata.appid, + install_dir=target, + username=args.username, + language=language, + max_downloads=args.max_downloads, + ) + + print("Reconciling with steamcmd (validate)...") + library = next(lib for lib in libraries if lib.common == target.parent) + result = recognize.run_steamcmd_validate(metadata.appid, target, args.username) + + if result.completed: + print("steamcmd finished cleanly -- Steam should show Play.") + return 0 + + if result.crashed_on_install_script: + print( + "steamcmd hit the known install-script crash; finishing the manifest manually..." + ) + manifest_path = recognize.finish_manifest_manually( + library=library, + appid=metadata.appid, + install_dir=target, + now_unix=int(time.time()), + ) + print(f"Wrote {manifest_path} -- launch Steam and confirm Play works.") + return 0 + + print( + "steamcmd failed for an unrecognized reason -- output above, not auto-resolved." + ) + print(result.output[-4000:]) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/_installer_import/steam_installer/depotdl.py b/_installer_import/steam_installer/depotdl.py new file mode 100644 index 0000000..cc3c60c --- /dev/null +++ b/_installer_import/steam_installer/depotdl.py @@ -0,0 +1,46 @@ +"""Wraps depotdownloader for the bulk content pull -- this is the fast path. + +No `-depot` filter is passed: depotdownloader's own per-depot ownership check +(AccountHasAccess, checked before any key fetch) already skips depots the +account doesn't own and continues with the rest, as long as `-os` is set +correctly. Without `-os`, it defaults to the host OS and can filter out every +real (Windows-tagged) depot, leaving only stray OS-agnostic ones -- that's +what caused the very first "0 bytes from 0 depots" failure this project was +built to avoid. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +DEFAULT_MAX_DOWNLOADS = 32 + + +def download( + appid: int, + install_dir: Path, + username: str, + language: str = "english", + os_name: str = "windows", + max_downloads: int = DEFAULT_MAX_DOWNLOADS, +) -> None: + """Run depotdownloader against `install_dir`, streaming its own progress output live.""" + install_dir.mkdir(parents=True, exist_ok=True) + cmd = [ + "depotdownloader", + "-app", + str(appid), + "-os", + os_name, + "-language", + language, + "-max-downloads", + str(max_downloads), + "-username", + username, + "-remember-password", + "-dir", + str(install_dir), + ] + subprocess.run(cmd, check=True) diff --git a/_installer_import/steam_installer/library.py b/_installer_import/steam_installer/library.py new file mode 100644 index 0000000..5694bd6 --- /dev/null +++ b/_installer_import/steam_installer/library.py @@ -0,0 +1,76 @@ +"""Steam library discovery: find install roots and where a given app lives.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from steam_installer import vdf + +DEFAULT_STEAM_ROOT = Path.home() / ".local/share/Steam" + + +@dataclass +class SteamLibrary: + path: Path + + @property + def common(self) -> Path: + return self.path / "steamapps" / "common" + + @property + def steamapps(self) -> Path: + return self.path / "steamapps" + + +def find_steam_root() -> Path: + """Locate the Steam installation. Raises if none of the usual paths exist.""" + for candidate in (DEFAULT_STEAM_ROOT, Path.home() / ".steam/steam"): + if (candidate / "steamapps").is_dir(): + return candidate + raise FileNotFoundError( + "No Steam installation found at ~/.local/share/Steam or ~/.steam/steam" + ) + + +def list_libraries(steam_root: Path) -> list[SteamLibrary]: + """Every Steam library folder, parsed from libraryfolders.vdf (plus steam_root itself).""" + libraries = [SteamLibrary(steam_root)] + vdf_path = steam_root / "steamapps" / "libraryfolders.vdf" + if not vdf_path.is_file(): + return libraries + data = vdf.parse(vdf_path.read_text()) + folders = data.get("libraryfolders", {}) + if not isinstance(folders, dict): + return libraries + seen = {steam_root.resolve()} + for entry in folders.values(): + if not isinstance(entry, dict): + continue + raw_path = entry.get("path") + if not raw_path: + continue + path = Path(raw_path).resolve() + if path not in seen: + seen.add(path) + libraries.append(SteamLibrary(path)) + return libraries + + +def find_existing_install( + libraries: list[SteamLibrary], installdir: str +) -> Path | None: + """Return the install path if `installdir` already exists in any library.""" + for library in libraries: + candidate = library.common / installdir + if candidate.is_dir(): + return candidate + return None + + +def pick_install_target(libraries: list[SteamLibrary], installdir: str) -> Path: + """Where to install: an existing partial install if one exists, else the primary library.""" + existing = find_existing_install(libraries, installdir) + if existing is not None: + return existing + return libraries[0].common / installdir diff --git a/_installer_import/steam_installer/recognize.py b/_installer_import/steam_installer/recognize.py new file mode 100644 index 0000000..e8bf511 --- /dev/null +++ b/_installer_import/steam_installer/recognize.py @@ -0,0 +1,137 @@ +"""Make the Steam client recognize downloaded content as installed. + +Primary path: `steamcmd ... app_update validate`. It hash-checks +existing files against the depot manifest and only fetches real deltas -- +confirmed live against a real 90+ GB install, going from ~24% to ~41% +credited in one pass without redownloading from scratch. + +Fallback: some titles ship a post-install script (installscript.vdf) that +steamcmd tries to run through a private, headless Proton setup. On at least +one real title this hit a confirmed steamcmd bug -- a stack-overflow segfault +in the closed-source steamclient.so error-handling path after the script +evaluator reports failure (crash backtrace: the same stack frame repeating +dozens of times). There's no known fix. When that specific crash is +detected, we finish the appmanifest by hand using ONLY values we already +verified ourselves: the exact manifest IDs from depotdownloader's own local +cache (never guessed), and the real on-disk size via `du`. Steam is left to +self-correct the rest (byte counters, staged-file bookkeeping) on its very +next launch -- confirmed: it did exactly that in practice, cleaning up the +hand-edited fields on its own. +""" + +from __future__ import annotations + +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from steam_installer import vdf +from steam_installer.library import SteamLibrary + +CRASH_SIGNATURE = "Standalone evaluator returned error code" + +MANIFEST_CACHE_RE = re.compile(r"^(\d+)_(\d+)\.manifest$") + + +class RecognizeError(Exception): + pass + + +@dataclass +class ValidateResult: + completed: bool + crashed_on_install_script: bool + output: str + + +def run_steamcmd_validate( + appid: int, install_dir: Path, username: str +) -> ValidateResult: + """Run steamcmd's app_update ... validate. `force_install_dir` must precede `+login` + (steamcmd itself warns about this ordering).""" + cmd = [ + "steamcmd", + "+force_install_dir", + str(install_dir), + "+login", + username, + "+app_update", + str(appid), + "validate", + "+quit", + ] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600) + output = result.stdout + result.stderr + crashed = CRASH_SIGNATURE in output + return ValidateResult( + completed=result.returncode == 0, + crashed_on_install_script=crashed, + output=output, + ) + + +def discover_cached_depot_manifests(install_dir: Path) -> dict[str, str]: + """Depot ID -> manifest ID, read from depotdownloader's own local cache filenames.""" + cache_dir = install_dir / ".DepotDownloader" + found: dict[str, str] = {} + if not cache_dir.is_dir(): + return found + for entry in cache_dir.iterdir(): + match = MANIFEST_CACHE_RE.match(entry.name) + if match: + depot_id, manifest_id = match.groups() + found[depot_id] = manifest_id + return found + + +def finish_manifest_manually( + library: SteamLibrary, + appid: int, + install_dir: Path, + now_unix: int, +) -> Path: + """Patch appmanifest_.acf to StateFlags=4 using only verified-known values. + + Requires steamcmd to have already written a partial manifest (even a crashed + run does) -- that's where buildid and SharedDepots come from. This never + fabricates a manifest from nothing. + """ + manifest_path = library.steamapps / f"appmanifest_{appid}.acf" + if not manifest_path.is_file(): + raise RecognizeError( + f"No existing {manifest_path.name} to finish -- run steamcmd's " + "app_update at least once first (even a crashed run writes one)." + ) + data = vdf.parse(manifest_path.read_text()) + app_state = data.get("AppState", {}) + if not isinstance(app_state, dict): + raise RecognizeError(f"{manifest_path} is not a valid AppState manifest") + + depot_manifests = discover_cached_depot_manifests(install_dir) + if not depot_manifests: + raise RecognizeError( + f"No cached depot manifests found under {install_dir}/.DepotDownloader" + ) + + size_on_disk = sum(f.stat().st_size for f in install_dir.rglob("*") if f.is_file()) + buildid = app_state.get("buildid") or app_state.get("TargetBuildID") or "0" + + installed_depots = { + depot_id: {"manifest": manifest_id, "size": "0"} + for depot_id, manifest_id in depot_manifests.items() + } + + app_state["StateFlags"] = "4" + app_state["LastUpdated"] = str(now_unix) + app_state["SizeOnDisk"] = str(size_on_disk) + app_state["StagingSize"] = "0" + app_state["buildid"] = buildid + app_state["UpdateResult"] = "0" + app_state["InstalledDepots"] = installed_depots + app_state.pop("StagedDepots", None) + + backup_path = manifest_path.with_suffix(".acf.bak") + backup_path.write_text(manifest_path.read_text()) + manifest_path.write_text(vdf.dumps("AppState", app_state)) + return manifest_path diff --git a/_installer_import/steam_installer/resolve.py b/_installer_import/steam_installer/resolve.py new file mode 100644 index 0000000..4e9c7e7 --- /dev/null +++ b/_installer_import/steam_installer/resolve.py @@ -0,0 +1,61 @@ +"""Resolve a game name to a Steam AppID via the public store-search endpoint. + +store.steampowered.com is known to TLS-reject scripted clients from at least +some networks (confirmed: even its bare homepage fails the TLS handshake for +plain curl/urllib here, while every other Steam domain works fine -- looks +like storefront-specific bot detection, not a general outage). Treat this as +best-effort: search() can fail, and callers should fall back to asking for a +numeric AppID directly rather than treating a search failure as fatal. +""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + +SEARCH_URL = "https://store.steampowered.com/api/storesearch/?term={term}&cc=us" + + +@dataclass +class SearchResult: + appid: int + name: str + + +class ResolveError(Exception): + pass + + +class SearchUnavailable(Exception): + """The store search endpoint couldn't be reached (network/TLS block, not a code bug).""" + + +def search(name: str) -> list[SearchResult]: + """Query Steam's store search API for candidate matches. No auth needed.""" + url = SEARCH_URL.format(term=urllib.parse.quote(name)) + try: + with urllib.request.urlopen(url, timeout=15) as response: + payload = json.loads(response.read()) + except (urllib.error.URLError, TimeoutError) as exc: + raise SearchUnavailable(str(exc)) from exc + items = payload.get("items", []) + return [SearchResult(appid=item["id"], name=item["name"]) for item in items] + + +def prompt_confirm(results: list[SearchResult]) -> SearchResult: + """Show numbered candidates and require an explicit choice. Never guesses.""" + if not results: + raise ResolveError("No matches found on Steam.") + print("Matches found:") + for i, result in enumerate(results, start=1): + print(f" {i}. {result.name} (appid {result.appid})") + while True: + choice = input(f"Which one? [1-{len(results)}, or 0 to cancel]: ").strip() + if choice == "0": + raise ResolveError("Cancelled by user.") + if choice.isdigit() and 1 <= int(choice) <= len(results): + return results[int(choice) - 1] + print("Invalid choice, try again.") diff --git a/_installer_import/steam_installer/vdf.py b/_installer_import/steam_installer/vdf.py new file mode 100644 index 0000000..11d282f --- /dev/null +++ b/_installer_import/steam_installer/vdf.py @@ -0,0 +1,120 @@ +"""Minimal reader/writer for Valve's KeyValues (VDF) text format. + +Steam uses this format for appmanifest_*.acf, libraryfolders.vdf, and +steamcmd's `app_info_print` output. Only the plain-text variant is +supported (no binary VDF) -- that covers every file this project touches. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +VdfValue = "str | dict[str, VdfValue]" + + +@dataclass +class VdfParseError(Exception): + message: str + position: int + + def __str__(self) -> str: + return f"{self.message} (at position {self.position})" + + +def parse(text: str) -> dict[str, VdfValue]: + """Parse VDF text into a nested dict. Returns the top-level block's contents.""" + tokens = _tokenize(text) + pos = 0 + + def parse_block() -> tuple[dict[str, VdfValue], int]: + nonlocal pos + result: dict[str, VdfValue] = {} + while pos < len(tokens): + token = tokens[pos] + if token == "}": + pos += 1 + return result, pos + key = token + pos += 1 + if pos >= len(tokens): + raise VdfParseError(f"unexpected end of input after key {key!r}", pos) + if tokens[pos] == "{": + pos += 1 + value, pos = parse_block() + else: + value = tokens[pos] + pos += 1 + result[key] = value + return result, pos + + # A VDF file is one root key followed by one root block. + if not tokens: + return {} + root_key = tokens[0] + pos = 1 + if pos >= len(tokens) or tokens[pos] != "{": + raise VdfParseError("expected '{' after root key", pos) + pos += 1 + root_value, pos = parse_block() + return {root_key: root_value} + + +def _tokenize(text: str) -> list[str]: + tokens: list[str] = [] + i = 0 + n = len(text) + while i < n: + ch = text[i] + if ch in " \t\r\n": + i += 1 + continue + if ch == "/" and i + 1 < n and text[i + 1] == "/": + while i < n and text[i] != "\n": + i += 1 + continue + if ch in "{}": + tokens.append(ch) + i += 1 + continue + if ch == '"': + j = i + 1 + buf = [] + while j < n and text[j] != '"': + if text[j] == "\\" and j + 1 < n: + buf.append(text[j + 1]) + j += 2 + else: + buf.append(text[j]) + j += 1 + tokens.append("".join(buf)) + i = j + 1 + continue + # Unquoted token (rare in Steam's own files, but tolerate it). + j = i + while j < n and text[j] not in " \t\r\n{}": + j += 1 + tokens.append(text[i:j]) + i = j + return tokens + + +def dumps(root_key: str, data: dict[str, VdfValue], indent: str = "\t") -> str: + """Serialize a dict back to VDF text, matching Steam's own tab-indented style.""" + lines = [f'"{root_key}"', "{"] + _dump_block(data, indent, 1, lines) + lines.append("}") + return "\n".join(lines) + "\n" + + +def _dump_block( + data: dict[str, VdfValue], indent: str, depth: int, lines: list[str] +) -> None: + prefix = indent * depth + for key, value in data.items(): + if isinstance(value, dict): + lines.append(f'{prefix}"{key}"') + lines.append(f"{prefix}{{") + _dump_block(value, indent, depth + 1, lines) + lines.append(f"{prefix}}}") + else: + lines.append(f'{prefix}"{key}"\t\t"{value}"') diff --git a/_installer_import/tests/test_library.py b/_installer_import/tests/test_library.py new file mode 100644 index 0000000..e6ec1c0 --- /dev/null +++ b/_installer_import/tests/test_library.py @@ -0,0 +1,70 @@ +from pathlib import Path + +from steam_installer.library import ( + SteamLibrary, + find_existing_install, + list_libraries, + pick_install_target, +) + + +def _make_steam_root(tmp_path: Path, extra_library: Path | None = None) -> Path: + root = tmp_path / "Steam" + (root / "steamapps").mkdir(parents=True) + if extra_library is not None: + (root / "steamapps" / "libraryfolders.vdf").write_text( + f""" +"libraryfolders" +{{ + "0" + {{ + "path" "{extra_library}" + }} +}} +""" + ) + return root + + +def test_list_libraries_includes_root_when_no_vdf(tmp_path): + root = _make_steam_root(tmp_path) + libraries = list_libraries(root) + assert [lib.path for lib in libraries] == [root] + + +def test_list_libraries_includes_extra_library_from_vdf(tmp_path): + extra = tmp_path / "SecondDrive" + root = _make_steam_root(tmp_path, extra_library=extra) + libraries = list_libraries(root) + assert {lib.path for lib in libraries} == {root, extra.resolve()} + + +def test_find_existing_install_locates_game_in_any_library(tmp_path): + root = _make_steam_root(tmp_path) + other = tmp_path / "Other" + (other / "steamapps" / "common" / "MyGame").mkdir(parents=True) + libraries = [SteamLibrary(root), SteamLibrary(other)] + found = find_existing_install(libraries, "MyGame") + assert found == other / "steamapps" / "common" / "MyGame" + + +def test_find_existing_install_returns_none_when_absent(tmp_path): + root = _make_steam_root(tmp_path) + libraries = [SteamLibrary(root)] + assert find_existing_install(libraries, "NoSuchGame") is None + + +def test_pick_install_target_prefers_existing_install(tmp_path): + root = _make_steam_root(tmp_path) + other = tmp_path / "Other" + (other / "steamapps" / "common" / "MyGame").mkdir(parents=True) + libraries = [SteamLibrary(root), SteamLibrary(other)] + target = pick_install_target(libraries, "MyGame") + assert target == other / "steamapps" / "common" / "MyGame" + + +def test_pick_install_target_defaults_to_primary_library(tmp_path): + root = _make_steam_root(tmp_path) + libraries = [SteamLibrary(root)] + target = pick_install_target(libraries, "NewGame") + assert target == root / "steamapps" / "common" / "NewGame" diff --git a/_installer_import/tests/test_recognize.py b/_installer_import/tests/test_recognize.py new file mode 100644 index 0000000..9645857 --- /dev/null +++ b/_installer_import/tests/test_recognize.py @@ -0,0 +1,81 @@ +from pathlib import Path + +import pytest + +from steam_installer import vdf +from steam_installer.library import SteamLibrary +from steam_installer.recognize import ( + RecognizeError, + discover_cached_depot_manifests, + finish_manifest_manually, +) + + +def test_discover_cached_depot_manifests_parses_filenames(tmp_path): + cache_dir = tmp_path / ".DepotDownloader" + cache_dir.mkdir() + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + (cache_dir / "1771302_257127247679847518.manifest.sha").write_bytes(b"x") + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + (cache_dir / "depot.config").write_bytes(b"x") + + found = discover_cached_depot_manifests(tmp_path) + assert found == { + "1771302": "257127247679847518", + "1771303": "1999516201078544527", + } + + +def test_discover_cached_depot_manifests_missing_dir_returns_empty(tmp_path): + assert discover_cached_depot_manifests(tmp_path) == {} + + +def _write_partial_manifest(steamapps: Path, appid: int) -> None: + steamapps.mkdir(parents=True, exist_ok=True) + data = { + "appid": str(appid), + "StateFlags": "1026", + "buildid": "23914554", + "SharedDepots": {"228989": "228980"}, + } + (steamapps / f"appmanifest_{appid}.acf").write_text(vdf.dumps("AppState", data)) + + +def test_finish_manifest_manually_requires_existing_manifest(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + install_dir = tmp_path / "Steam" / "steamapps" / "common" / "Game" + install_dir.mkdir(parents=True) + with pytest.raises(RecognizeError, match="run steamcmd"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + +def test_finish_manifest_manually_requires_cached_depot_manifests(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + install_dir.mkdir(parents=True) + with pytest.raises(RecognizeError, match="No cached depot manifests"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + +def test_finish_manifest_manually_sets_state_flags_and_depots(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + (install_dir / "game.bin").write_bytes(b"0123456789") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + result = vdf.parse(manifest_path.read_text()) + app_state = result["AppState"] + assert app_state["StateFlags"] == "4" + assert app_state["buildid"] == "23914554" + assert app_state["InstalledDepots"]["1771302"]["manifest"] == "257127247679847518" + assert app_state["SharedDepots"] == {"228989": "228980"} + assert "StagedDepots" not in app_state + assert manifest_path.with_suffix(".acf.bak").is_file() diff --git a/_installer_import/tests/test_vdf.py b/_installer_import/tests/test_vdf.py new file mode 100644 index 0000000..e742f07 --- /dev/null +++ b/_installer_import/tests/test_vdf.py @@ -0,0 +1,68 @@ +from steam_installer import vdf + + +def test_parse_flat_block(): + text = """ +"AppState" +{ + "appid" "1771300" + "name" "Kingdom Come: Deliverance II" +} +""" + result = vdf.parse(text) + assert result == { + "AppState": { + "appid": "1771300", + "name": "Kingdom Come: Deliverance II", + } + } + + +def test_parse_nested_blocks(): + text = """ +"AppState" +{ + "InstalledDepots" + { + "1771302" + { + "manifest" "257127247679847518" + "size" "199419496" + } + } +} +""" + result = vdf.parse(text) + assert ( + result["AppState"]["InstalledDepots"]["1771302"]["manifest"] + == "257127247679847518" + ) + + +def test_parse_handles_comments_and_blank_lines(): + text = """ +// leading comment +"AppState" +{ + + "appid" "1" // trailing comment +} +""" + result = vdf.parse(text) + assert result == {"AppState": {"appid": "1"}} + + +def test_roundtrip_dumps_then_parse(): + data = { + "appid": "1771300", + "InstalledDepots": { + "1771302": {"manifest": "257127247679847518", "size": "199419496"}, + }, + } + text = vdf.dumps("AppState", data) + reparsed = vdf.parse(text) + assert reparsed == {"AppState": data} + + +def test_empty_input_returns_empty_dict(): + assert vdf.parse("") == {} From f9e91bba178d871a5dd734e74f2cdc3077c7f56b Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 25 Jul 2026 15:51:20 +0200 Subject: [PATCH 02/24] fix(recognize): verify manifest state instead of trusting steamcmd's exit code Confirmed live against a real install: steamcmd's app_update can exit 0 without actually finishing -- it left installdir pointing at a stale, unrelated folder name (from an earlier corrupted manifest state) and InstalledDepots empty despite a "clean" exit. Relaunching the Steam client then read that same stale field and wiped the correctly-downloaded folder as orphaned while starting a fresh download into the wrong one -- real data loss, not a hypothetical. - manifest_reflects_real_install() checks StateFlags/installdir/ InstalledDepots directly instead of trusting the return code. - ensure_installdir_correct() patches a stale installdir BEFORE running steamcmd, not just in the crash-fallback path after. - run_steamcmd_validate() no longer captures stdout/stderr: steamcmd prompts for a password (and Steam Guard) on stdin, and prompt text written to a captured/buffered stdout never reached the terminal -- the process looked hung with no visible request for input. Output now inherits the parent terminal, matching depotdownloader's own interactive login. The confirmed install-script segfault is detected via the SIGSEGV exit signal instead of grepping captured text. - finish_manifest_manually() now also corrects installdir (previously only fixed by ensure_installdir_correct, not the crash-fallback path) and no longer lets a "0" buildid sentinel short-circuit past a real TargetBuildID via a bare `or` chain. - cli.py gains --skip-steamcmd: go straight from depotdownloader's download to a direct manifest write, bypassing steamcmd's app_update entirely. It has now shown two separate failure modes (the segfault, and a silent no-op run) -- this is the same direct-write approach already confirmed to work standalone. Co-Authored-By: Claude Sonnet 5 --- _installer_import/steam_installer/cli.py | 67 ++++++--- .../steam_installer/recognize.py | 88 ++++++++++-- _installer_import/tests/test_recognize.py | 135 ++++++++++++++++++ 3 files changed, 259 insertions(+), 31 deletions(-) diff --git a/_installer_import/steam_installer/cli.py b/_installer_import/steam_installer/cli.py index c32e257..1bf92fc 100644 --- a/_installer_import/steam_installer/cli.py +++ b/_installer_import/steam_installer/cli.py @@ -55,13 +55,25 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--max-downloads", type=int, default=depotdl.DEFAULT_MAX_DOWNLOADS ) + parser.add_argument( + "--skip-steamcmd", + action="store_true", + help=( + "Finish the manifest directly from depotdownloader's own cached data " + "instead of running steamcmd's app_update. Steamcmd's install-script " + "step has shown two separate failure modes (a confirmed segfault, " + "and a silent no-op run); this is the same direct-write approach " + "already confirmed to work." + ), + ) args = parser.parse_args(argv) if args.appid is None and args.name is None: parser.error("provide a game name to search for, or --appid directly") _require_tool("depotdownloader", "See ~/depotdownloader.") - _require_tool("steamcmd", "Run: yay -S steamcmd") + if not args.skip_steamcmd: + _require_tool("steamcmd", "Run: yay -S steamcmd") appid = _resolve_appid(args) metadata = appinfo.get_metadata(appid) @@ -89,32 +101,43 @@ def main(argv: list[str] | None = None) -> int: max_downloads=args.max_downloads, ) - print("Reconciling with steamcmd (validate)...") library = next(lib for lib in libraries if lib.common == target.parent) - result = recognize.run_steamcmd_validate(metadata.appid, target, args.username) + recognize.ensure_installdir_correct(library, metadata.appid, target) - if result.completed: - print("steamcmd finished cleanly -- Steam should show Play.") - return 0 + if not args.skip_steamcmd: + print("Reconciling with steamcmd (validate)...") + result = recognize.run_steamcmd_validate(metadata.appid, target, args.username) - if result.crashed_on_install_script: - print( - "steamcmd hit the known install-script crash; finishing the manifest manually..." - ) - manifest_path = recognize.finish_manifest_manually( - library=library, - appid=metadata.appid, - install_dir=target, - now_unix=int(time.time()), - ) - print(f"Wrote {manifest_path} -- launch Steam and confirm Play works.") - return 0 + # Never trust steamcmd's exit code alone -- verify the manifest actually + # reflects a real install before believing it. A "clean" exit has + # produced a manifest steamcmd itself hadn't finished updating before. + if ( + not result.crashed_on_install_script + and recognize.manifest_reflects_real_install( + library, metadata.appid, target + ) + ): + print("steamcmd finished cleanly -- Steam should show Play.") + return 0 - print( - "steamcmd failed for an unrecognized reason -- output above, not auto-resolved." + if result.crashed_on_install_script: + print( + "steamcmd hit the known install-script crash; finishing the manifest manually..." + ) + else: + print( + f"steamcmd exited {result.returncode} but the manifest doesn't reflect a " + "real install; finishing it manually..." + ) + + manifest_path = recognize.finish_manifest_manually( + library=library, + appid=metadata.appid, + install_dir=target, + now_unix=int(time.time()), ) - print(result.output[-4000:]) - return 1 + print(f"Wrote {manifest_path} -- launch Steam and confirm Play works.") + return 0 if __name__ == "__main__": diff --git a/_installer_import/steam_installer/recognize.py b/_installer_import/steam_installer/recognize.py index e8bf511..f71c889 100644 --- a/_installer_import/steam_installer/recognize.py +++ b/_installer_import/steam_installer/recognize.py @@ -22,6 +22,7 @@ hand-edited fields on its own. from __future__ import annotations import re +import signal import subprocess from dataclasses import dataclass from pathlib import Path @@ -29,8 +30,6 @@ from pathlib import Path from steam_installer import vdf from steam_installer.library import SteamLibrary -CRASH_SIGNATURE = "Standalone evaluator returned error code" - MANIFEST_CACHE_RE = re.compile(r"^(\d+)_(\d+)\.manifest$") @@ -42,14 +41,28 @@ class RecognizeError(Exception): class ValidateResult: completed: bool crashed_on_install_script: bool - output: str + returncode: int def run_steamcmd_validate( appid: int, install_dir: Path, username: str ) -> ValidateResult: """Run steamcmd's app_update ... validate. `force_install_dir` must precede `+login` - (steamcmd itself warns about this ordering).""" + (steamcmd itself warns about this ordering). + + Deliberately does NOT capture stdout/stderr: steamcmd prompts for a + password (and possibly a Steam Guard code) on stdin, and prompt text + written to a captured/buffered stdout never reaches the terminal -- + the process looks hung with no visible request for input. Output is + left to inherit the parent's terminal instead, exactly like + depotdownloader's own interactive login. + + The confirmed steamcmd install-script bug (see module docstring) kills + the process with SIGSEGV -- Python reports that as returncode -11 on + POSIX. Checking the signal directly is more reliable than grepping + captured text for a message string, and doesn't require capturing + output at all. + """ cmd = [ "steamcmd", "+force_install_dir", @@ -61,13 +74,64 @@ def run_steamcmd_validate( "validate", "+quit", ] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600) - output = result.stdout + result.stderr - crashed = CRASH_SIGNATURE in output + result = subprocess.run(cmd, timeout=3600) + crashed = result.returncode == -signal.SIGSEGV return ValidateResult( completed=result.returncode == 0, crashed_on_install_script=crashed, - output=output, + returncode=result.returncode, + ) + + +def ensure_installdir_correct( + library: SteamLibrary, appid: int, install_dir: Path +) -> None: + """Patch a stale `installdir` field BEFORE running steamcmd, not just after. + + Confirmed necessary: a prior, unrelated Steam GUI glitch left this app's + manifest pointing at a different folder name. steamcmd's own validate run + didn't correct it despite a clean exit, and relaunching the Steam client + read that same stale field and wiped the correctly-downloaded folder as + "orphaned" while starting a fresh download into the wrong one. Fixing it + proactively, before steamcmd ever runs, closes that window entirely. + Does nothing if no manifest exists yet -- steamcmd creates one itself in + that case, with the right installdir since we pass force_install_dir. + """ + manifest_path = library.steamapps / f"appmanifest_{appid}.acf" + if not manifest_path.is_file(): + return + data = vdf.parse(manifest_path.read_text()) + app_state = data.get("AppState", {}) + if not isinstance(app_state, dict): + return + if app_state.get("installdir") == install_dir.name: + return + app_state["installdir"] = install_dir.name + manifest_path.write_text(vdf.dumps("AppState", app_state)) + + +def manifest_reflects_real_install( + library: SteamLibrary, appid: int, install_dir: Path +) -> bool: + """Don't trust steamcmd's exit code alone -- verify the manifest actually matches + reality. Confirmed necessary the hard way: steamcmd exited 0 while leaving + `installdir` pointed at a stale, unrelated folder name from an earlier corrupted + manifest state, `InstalledDepots` empty, and `StateFlags` still "update needed" -- + a "clean" exit that hadn't actually recognized anything. Relaunching the Steam + client then treated the real, correctly-downloaded folder as orphaned and wiped + it while starting a fresh (slow, wrong-folder) download. + """ + manifest_path = library.steamapps / f"appmanifest_{appid}.acf" + if not manifest_path.is_file(): + return False + data = vdf.parse(manifest_path.read_text()) + app_state = data.get("AppState", {}) + if not isinstance(app_state, dict): + return False + return ( + app_state.get("StateFlags") == "4" + and app_state.get("installdir") == install_dir.name + and bool(app_state.get("InstalledDepots")) ) @@ -115,7 +179,12 @@ def finish_manifest_manually( ) size_on_disk = sum(f.stat().st_size for f in install_dir.rglob("*") if f.is_file()) - buildid = app_state.get("buildid") or app_state.get("TargetBuildID") or "0" + # "0" means "no real build installed yet" (steamcmd/Steam use it as a + # sentinel) -- a bare `or` treats that non-empty string as truthy and + # never falls through to TargetBuildID, leaving a stale/wrong buildid. + existing_buildid = app_state.get("buildid") + buildid = existing_buildid if existing_buildid and existing_buildid != "0" else None + buildid = buildid or app_state.get("TargetBuildID") or "0" installed_depots = { depot_id: {"manifest": manifest_id, "size": "0"} @@ -123,6 +192,7 @@ def finish_manifest_manually( } app_state["StateFlags"] = "4" + app_state["installdir"] = install_dir.name app_state["LastUpdated"] = str(now_unix) app_state["SizeOnDisk"] = str(size_on_disk) app_state["StagingSize"] = "0" diff --git a/_installer_import/tests/test_recognize.py b/_installer_import/tests/test_recognize.py index 9645857..6a318cf 100644 --- a/_installer_import/tests/test_recognize.py +++ b/_installer_import/tests/test_recognize.py @@ -7,7 +7,9 @@ from steam_installer.library import SteamLibrary from steam_installer.recognize import ( RecognizeError, discover_cached_depot_manifests, + ensure_installdir_correct, finish_manifest_manually, + manifest_reflects_real_install, ) @@ -79,3 +81,136 @@ def test_finish_manifest_manually_sets_state_flags_and_depots(tmp_path): assert app_state["SharedDepots"] == {"228989": "228980"} assert "StagedDepots" not in app_state assert manifest_path.with_suffix(".acf.bak").is_file() + + +def test_finish_manifest_manually_falls_back_to_target_buildid_when_zero(tmp_path): + """buildid "0" is a sentinel for "no real build installed yet", not a real + value -- a naive `or` chain treats it as truthy and never falls through.""" + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + data = { + "appid": "1771300", + "StateFlags": "1026", + "buildid": "0", + "TargetBuildID": "23914554", + } + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data) + ) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert app_state["buildid"] == "23914554" + + +def test_finish_manifest_manually_corrects_installdir(tmp_path): + """The exact bug that caused real data loss: a stale installdir field + pointing at a different folder than where content was actually downloaded.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + data = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text()) + data["AppState"]["installdir"] = "Some Stale Wrong Name" + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data["AppState"]) + ) + + install_dir = library.common / "KingdomComeDeliverance2" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert app_state["installdir"] == "KingdomComeDeliverance2" + + +def test_ensure_installdir_correct_patches_stale_field(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + data = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text()) + data["AppState"]["installdir"] = "Kingdom Come: Deliverance II" + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data["AppState"]) + ) + + install_dir = library.common / "KingdomComeDeliverance2" + ensure_installdir_correct(library, 1771300, install_dir) + + app_state = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text())[ + "AppState" + ] + assert app_state["installdir"] == "KingdomComeDeliverance2" + + +def test_ensure_installdir_correct_noop_when_already_correct(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + manifest_path = library.steamapps / "appmanifest_1771300.acf" + data = vdf.parse(manifest_path.read_text()) + data["AppState"]["installdir"] = "KingdomComeDeliverance2" + manifest_path.write_text(vdf.dumps("AppState", data["AppState"])) + before = manifest_path.read_text() + + ensure_installdir_correct( + library, 1771300, library.common / "KingdomComeDeliverance2" + ) + + assert manifest_path.read_text() == before + + +def test_ensure_installdir_correct_noop_when_no_manifest(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + # Should not raise, and should not create a manifest out of nothing. + ensure_installdir_correct(library, 1771300, library.common / "Game") + assert not (library.steamapps / "appmanifest_1771300.acf").exists() + + +def test_manifest_reflects_real_install_true_when_consistent(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + data = { + "StateFlags": "4", + "installdir": "KingdomComeDeliverance2", + "InstalledDepots": {"1771302": {"manifest": "257127247679847518"}}, + } + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data) + ) + + assert manifest_reflects_real_install( + library, 1771300, library.common / "KingdomComeDeliverance2" + ) + + +def test_manifest_reflects_real_install_false_when_installdir_mismatches(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + data = { + "StateFlags": "4", + "installdir": "Kingdom Come: Deliverance II", + "InstalledDepots": {"1771302": {"manifest": "257127247679847518"}}, + } + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data) + ) + + assert not manifest_reflects_real_install( + library, 1771300, library.common / "KingdomComeDeliverance2" + ) + + +def test_manifest_reflects_real_install_false_when_no_manifest(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + assert not manifest_reflects_real_install(library, 1771300, library.common / "Game") From aaf9c18a8a012500953f8f449b0ea102d91a404d Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Wed, 5 Aug 2026 10:11:34 +0200 Subject: [PATCH 03/24] fix(recognize): stop Steam re-downloading after a hand-finished install The KCD2 run reported success but Steam re-downloaded 90 GB anyway. Two independent causes, plus a defect that hid both. force_install_dir sets a Steam *library root*, not an install directory. Pointing it at the game folder made steamcmd search /steamapps/ for manifests ("Loaded 0 apps from install folder"), create that stray dir, and fail the app with "Invalid platform" for lack of depot context. Dropped it; the app already lives in a registered library. Separately, a Linux steamcmd needs +@sSteamCmdForcePlatformType windows for a Windows-only title. Steam then discarded the manifest because it was self-contradictory: StateFlags=4 while BytesDownloaded (22 GB) != BytesToDownload (92 GB), with tens of GB still staged under steamapps/downloading/. Across every genuinely installed manifest the byte counters are equal in pairs, StagingSize is 0 and staging is empty; DownloadType is kept, not dropped. Settle all of it, and refuse to call an install real until it holds. The defect that hid this: steamcmd exited 8 having done nothing and the tool still wrote StateFlags=4, printed a success line and returned 0. Now gated on depotdownloader succeeding, every required depot being cached, real game content on disk, a matching buildid and no running Steam client -- and it re-reads the manifest after writing. Also fixes data corruption found in review: vdf.dumps never re-escaped backslashes, so each parse/dump cycle halved them and silently mangled the Windows paths in appmanifest_228980.acf's InstallScripts. Manifest writes are now atomic with a preserved pristine .acf.bak. Adds per-run logging to ~/.local/state/steam-game-installer (argv, exit codes, full subprocess output, manifest before/after), teed through a pty so steamcmd's password prompt still works. Verified end-to-end on KCD2: Steam logged "Fully Installed, App Running" and the game launched with no download queued. Coverage 100% statements+branches, enforced by fail_under. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EK4urCmbTqN4Vrc3nGBcXF --- _installer_import/.gitignore | 2 + _installer_import/README.md | 105 ++++ _installer_import/install.sh | 26 + _installer_import/pyproject.toml | 15 + _installer_import/steam_installer/appinfo.py | 83 ++- _installer_import/steam_installer/cli.py | 191 +++++- _installer_import/steam_installer/depotdl.py | 17 +- .../steam_installer/recognize.py | 330 ++++++++++- _installer_import/steam_installer/runlog.py | 223 +++++++ _installer_import/steam_installer/vdf.py | 18 +- _installer_import/tests/conftest.py | 21 + _installer_import/tests/test_appinfo.py | 236 ++++++++ _installer_import/tests/test_cli.py | 328 +++++++++++ _installer_import/tests/test_depotdl.py | 80 +++ _installer_import/tests/test_library.py | 86 +++ _installer_import/tests/test_recognize.py | 551 +++++++++++++++++- _installer_import/tests/test_resolve.py | 99 ++++ _installer_import/tests/test_runlog.py | 210 +++++++ _installer_import/tests/test_vdf.py | 79 +++ 19 files changed, 2642 insertions(+), 58 deletions(-) create mode 100644 _installer_import/README.md create mode 100644 _installer_import/steam_installer/runlog.py create mode 100644 _installer_import/tests/conftest.py create mode 100644 _installer_import/tests/test_appinfo.py create mode 100644 _installer_import/tests/test_cli.py create mode 100644 _installer_import/tests/test_depotdl.py create mode 100644 _installer_import/tests/test_resolve.py create mode 100644 _installer_import/tests/test_runlog.py diff --git a/_installer_import/.gitignore b/_installer_import/.gitignore index 76180b2..472098a 100644 --- a/_installer_import/.gitignore +++ b/_installer_import/.gitignore @@ -3,3 +3,5 @@ __pycache__/ *.pyc .pytest_cache/ *.egg-info/ +.coverage +.coverage.* diff --git a/_installer_import/README.md b/_installer_import/README.md new file mode 100644 index 0000000..550e0ee --- /dev/null +++ b/_installer_import/README.md @@ -0,0 +1,105 @@ +# steam-game-installer + +Download an owned Steam game with `depotdownloader` (which saturates a link far +better than the Steam client), then make the Steam client accept the result as +a real install so **Play** works. + +```bash +./install.sh # checks depotdownloader + steamcmd are present +pip install -e . +steam-game-installer --appid 1771300 --username +``` + +**Close the Steam client first.** Steam owns `appmanifest_.acf` while it +runs and rewrites it from its own in-memory state, so anything written +underneath a live client is discarded. The tool refuses to start if it sees one. + +## Run logs + +Every run writes a full log: + +``` +~/.local/state/steam-game-installer/run--.log +~/.local/state/steam-game-installer/latest.log -> most recent +``` + +The path is printed at the start and end of every run, and on every error path. +It contains the full argv, exit code and complete output of each subprocess, the +manifest verbatim before and after every write, the depot cache listing, +expected-vs-actual depots, and whether Steam was running. Send `latest.log` when +something goes wrong. `--verbose` mirrors all of it to the console. + +Subprocess output is teed through a pty rather than captured, because steamcmd +only prompts for a password / Steam Guard code when it believes it is on a +terminal — with a plain pipe the prompt never appears and the run looks hung. + +## How it works + +1. `depotdownloader` pulls the content (fast path). +2. `steamcmd app_update validate` reconciles Steam's own bookkeeping. +3. If that fails or leaves the manifest incomplete, the manifest is finished by + hand — but only when every precondition below holds. + +## Fail-closed preconditions + +The tool refuses to mark an app installed, and exits non-zero, unless: + +- `depotdownloader` exited 0; +- every required depot has a cached manifest under `.DepotDownloader/` + (required = this app's depots for the chosen language and OS, excluding DLC, + optional and shared depots — KCD2 lists 12 depots but an english/windows + install only ever fetches 3); +- the manifest carries a real `buildid` matching the current public build; +- no Steam client is running; +- the manifest reads back as a real install *after* being written. + +This matters: an earlier version wrote `StateFlags=4` and reported success after +a steamcmd run that had done nothing at all. + +## Failure modes this encodes + +Each of these cost a real debugging session; the tests pin them. + +**`force_install_dir` is a library root, not an install directory.** Pointing it +at the game folder made steamcmd look for manifests in `/steamapps/`, find +none (`Loaded 0 apps from install folder`), create that stray directory, and then +fail the app with `Invalid platform` — it had no depot context for the app. +Dropped entirely: the app already lives in a registered library. + +**A Linux steamcmd needs `+@sSteamCmdForcePlatformType windows`** (before +`+login`) for a Windows-only game, or it resolves depots against the host OS. + +**Steam rejects an internally inconsistent manifest.** A file that says +`StateFlags=4` while `BytesDownloaded` (22 GB) still differs from +`BytesToDownload` (92 GB) is self-contradictory; Steam discards the "installed" +claim and resumes downloading. Across every genuinely installed manifest the +counters are equal in pairs and `StagingSize` is `0`. `DownloadType` is *kept* — +it is present on every real install, so removing it is not the fix. + +**Staging must be cleared too.** Steam stages chunks in +`steamapps/downloading//`. Declaring the app installed while tens of GB +sit there pending makes Steam resume the update. Installed apps have an empty +(4 KB) staging dir. + +**Shared depots are not installed depots.** `228989` belongs to app 228980 +(Steamworks Common Redistributables); listing it in `InstalledDepots` +contradicts `SharedDepots`. + +**`SizeOnDisk` excludes `.DepotDownloader/`** — its cache and staging tree are +not game content and overstate the total by gigabytes. + +## Development + +```bash +./install.sh # also prepares the test env +.venv/bin/python -m pytest tests/ -q +ruff check steam_installer/ tests/ +ruff format --check steam_installer/ tests/ +``` + +Coverage is enforced at **100% of statements and branches** +(`fail_under = 100`), so an uncovered line fails the suite rather than quietly +lowering a number. One test runs `python -m steam_installer.cli` in a +subprocess to prove the `__main__` entry point works; recording coverage from +a child process needs a `.pth` that calls `coverage.process_startup()`, which +`install.sh` writes into the venv. Skip `install.sh` and coverage reports 99%. diff --git a/_installer_import/install.sh b/_installer_import/install.sh index 54cd179..e41cbe9 100755 --- a/_installer_import/install.sh +++ b/_installer_import/install.sh @@ -5,6 +5,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR + missing=0 if ! command -v depotdownloader >/dev/null 2>&1; then @@ -17,8 +20,31 @@ if ! command -v steamcmd >/dev/null 2>&1; then missing=1 fi +setup_dev_env() { + # The test suite requires 100% coverage (pyproject: fail_under = 100), and + # one test runs `python -m steam_installer.cli` in a subprocess. Coverage in + # a child process is only recorded if it calls coverage.process_startup() at + # interpreter start, which is what this .pth does. Without it a clean clone + # reports 99% and the suite fails. + local venv="${SCRIPT_DIR}/.venv" + [[ -d "$venv" ]] || return 0 + + local python="${venv}/bin/python" + "$python" -m pip install -q -e "${SCRIPT_DIR}[dev]" 2>/dev/null || + "$python" -m pip install -q -e "$SCRIPT_DIR" pytest pytest-cov + + local site_packages + site_packages="$("$python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" + cat >"${site_packages}/steam_installer_coverage.pth" <<-'PTH' + import os, coverage; os.environ.get("COVERAGE_PROCESS_START") and coverage.process_startup() + PTH + echo "Dev environment ready (subprocess coverage enabled)." +} + if [[ "$missing" -eq 0 ]]; then echo "All dependencies present." fi +setup_dev_env + exit "$missing" diff --git a/_installer_import/pyproject.toml b/_installer_import/pyproject.toml index 6c9315a..43b0307 100644 --- a/_installer_import/pyproject.toml +++ b/_installer_import/pyproject.toml @@ -14,3 +14,18 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] testpaths = ["tests"] + +[tool.coverage.run] +source = ["steam_installer"] +branch = true +# The `__main__` guard is exercised by running the module as a script in a +# subprocess; without this its coverage would be recorded in a child process +# and thrown away. +parallel = true +concurrency = ["multiprocessing"] + +[tool.coverage.report] +# The suite covers every line and branch. Anything less is a regression, so +# make it fail rather than merely reporting a lower number. +fail_under = 100 +show_missing = true diff --git a/_installer_import/steam_installer/appinfo.py b/_installer_import/steam_installer/appinfo.py index 9a9a081..a238382 100644 --- a/_installer_import/steam_installer/appinfo.py +++ b/_installer_import/steam_installer/appinfo.py @@ -20,9 +20,37 @@ class AppMetadata: installdir: str +@dataclass +class DepotPlan: + """Which depots a given (os, language) install is actually supposed to contain. + + Needed to check that a download finished. "Every depot in app_info" is the + wrong bar: KCD2 lists 12 depots but an english/windows install only ever + fetches 3 of them, the rest being other languages, DLC and other platforms. + """ + + required: set[str] + shared: set[str] + + @property + def all_expected(self) -> set[str]: + return self.required | self.shared + + def _fetch_app_block(appid: int) -> dict: result = subprocess.run( - ["steamcmd", "+login", "anonymous", "+app_info_print", str(appid), "+quit"], + [ + "steamcmd", + # Without this, a Linux steamcmd resolves a Windows-only app's depot + # list against the host platform and can come back empty. + "+@sSteamCmdForcePlatformType", + "windows", + "+login", + "anonymous", + "+app_info_print", + str(appid), + "+quit", + ], capture_output=True, text=True, timeout=60, @@ -59,3 +87,56 @@ def get_metadata(appid: int) -> AppMetadata: raise AppInfoError(f"no installdir field found for app {appid}") return AppMetadata(appid=appid, name=name, installdir=installdir) + + +def get_build_id(appid: int, branch: str = "public") -> str | None: + """Current buildid for `branch`, used to sanity-check the manifest we write.""" + app_block = _fetch_app_block(appid) + depots = app_block.get("depots", {}) + if not isinstance(depots, dict): + return None + branches = depots.get("branches", {}) + if not isinstance(branches, dict): + return None + entry = branches.get(branch, {}) + if not isinstance(entry, dict): + return None + build_id = entry.get("buildid") + return build_id if isinstance(build_id, str) else None + + +def get_depot_plan( + appid: int, language: str = "english", os_name: str = "windows" +) -> DepotPlan: + """Depots an (os, language) install should end up with. + + Skips depots belonging to another language, another OS, a DLC (`dlcappid`) + or marked `optional` -- none of those are fetched by a default install, so + demanding them would make a healthy download look incomplete. + """ + app_block = _fetch_app_block(appid) + depots = app_block.get("depots", {}) + if not isinstance(depots, dict): + raise AppInfoError(f"no 'depots' section in app info for {appid}") + + required: set[str] = set() + shared: set[str] = set() + for depot_id, entry in depots.items(): + if not depot_id.isdigit() or not isinstance(entry, dict): + continue + # A shared depot is installed once under its owning app, not here. + if "depotfromapp" in entry or "sharedinstall" in entry: + shared.add(depot_id) + continue + if entry.get("dlcappid") or entry.get("optional") == "1": + continue + config = entry.get("config", {}) + config = config if isinstance(config, dict) else {} + oslist = config.get("oslist") + if isinstance(oslist, str) and oslist and os_name not in oslist.split(","): + continue + depot_language = config.get("language") + if isinstance(depot_language, str) and depot_language != language: + continue + required.add(depot_id) + return DepotPlan(required=required, shared=shared) diff --git a/_installer_import/steam_installer/cli.py b/_installer_import/steam_installer/cli.py index 1bf92fc..7cbb849 100644 --- a/_installer_import/steam_installer/cli.py +++ b/_installer_import/steam_installer/cli.py @@ -6,9 +6,15 @@ import argparse import shutil import sys import time +from pathlib import Path -from steam_installer import appinfo, depotdl, recognize, resolve -from steam_installer.library import find_steam_root, list_libraries, pick_install_target +from steam_installer import appinfo, depotdl, recognize, resolve, runlog +from steam_installer.library import ( + SteamLibrary, + find_steam_root, + list_libraries, + pick_install_target, +) def _require_tool(name: str, install_hint: str) -> None: @@ -40,7 +46,7 @@ def _resolve_appid(args: argparse.Namespace) -> int: return chosen.appid -def main(argv: list[str] | None = None) -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Download an owned Steam game fast, then make Steam recognize it." ) @@ -66,47 +72,115 @@ def main(argv: list[str] | None = None) -> int: "already confirmed to work." ), ) + parser.add_argument( + "--verbose", + action="store_true", + help="Show on the console everything that goes to the run log.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() args = parser.parse_args(argv) if args.appid is None and args.name is None: parser.error("provide a game name to search for, or --appid directly") + log = runlog.setup(args.appid, verbose=args.verbose) + logger = log.logger + # Printed up front as well as at the end: if the run dies badly, the path + # still has to be on screen to copy-paste. + logger.info("Run log: %s", log.path) + logger.debug("argv: %s", sys.argv) + + try: + return _run(args, log) + except Exception: + logger.exception("Run failed with an unhandled error") + logger.info("Run log (send this): %s", log.path) + return 1 + + +def _run(args: argparse.Namespace, log: runlog.RunLog) -> int: + logger = log.logger + _require_tool("depotdownloader", "See ~/depotdownloader.") if not args.skip_steamcmd: _require_tool("steamcmd", "Run: yay -S steamcmd") appid = _resolve_appid(args) metadata = appinfo.get_metadata(appid) - print(f"Confirmed: {metadata.name} (appid {metadata.appid})") + logger.info("Confirmed: %s (appid %s)", metadata.name, metadata.appid) answer = input("Proceed? [y/N]: ").strip().lower() if answer != "y": - print("Cancelled.") + logger.info("Cancelled.") return 1 steam_root = find_steam_root() libraries = list_libraries(steam_root) target = pick_install_target(libraries, metadata.installdir) - print(f"Install target: {target}") + logger.info("Install target: %s", target) language = args.language if language is None: language = input("Language [english]: ").strip() or "english" - print("Downloading via depotdownloader (fast path)...") - depotdl.download( + # Refuse early rather than after an hour of downloading: a live Steam client + # rewrites the manifest from its own state and discards whatever we write. + running = recognize.steam_client_pids() + if running: + logger.error( + "The Steam client is running (pid %s). It owns appmanifest_%s.acf and " + "will overwrite anything this tool writes. Close Steam, then re-run.", + ", ".join(str(p) for p in running), + metadata.appid, + ) + logger.info("Run log (send this): %s", log.path) + return 1 + + depot_plan = appinfo.get_depot_plan(metadata.appid, language) + expected_build_id = appinfo.get_build_id(metadata.appid) + log.section("Plan") + logger.debug("language=%s max_downloads=%s", language, args.max_downloads) + logger.debug("required depots: %s", sorted(depot_plan.required)) + logger.debug("shared depots: %s", sorted(depot_plan.shared)) + logger.debug("expected buildid: %s", expected_build_id) + + library = next(lib for lib in libraries if lib.common == target.parent) + manifest_path = library.steamapps / f"appmanifest_{metadata.appid}.acf" + log.file_contents("Manifest before run", manifest_path) + + logger.info("Downloading via depotdownloader (fast path)...") + log.section("depotdownloader") + returncode = depotdl.download( appid=metadata.appid, install_dir=target, username=args.username, language=language, max_downloads=args.max_downloads, + logger=logger, ) + if returncode != 0: + logger.error( + "depotdownloader failed (exit %d) -- not touching the manifest.", returncode + ) + logger.info("Run log (send this): %s", log.path) + return 1 + + log.section("depotdownloader cache") + for depot_id, mid in sorted( + recognize.discover_cached_depot_manifests(target).items() + ): + logger.debug("depot %s -> manifest %s", depot_id, mid) - library = next(lib for lib in libraries if lib.common == target.parent) recognize.ensure_installdir_correct(library, metadata.appid, target) if not args.skip_steamcmd: - print("Reconciling with steamcmd (validate)...") - result = recognize.run_steamcmd_validate(metadata.appid, target, args.username) + logger.info("Reconciling with steamcmd (validate)...") + log.section("steamcmd app_update validate") + result = recognize.run_steamcmd_validate(metadata.appid, args.username, logger) + log.file_contents("Manifest after steamcmd", manifest_path) # Never trust steamcmd's exit code alone -- verify the manifest actually # reflects a real install before believing it. A "clean" exit has @@ -117,26 +191,107 @@ def main(argv: list[str] | None = None) -> int: library, metadata.appid, target ) ): - print("steamcmd finished cleanly -- Steam should show Play.") + logger.info("steamcmd finished cleanly -- Steam should show Play.") + logger.info("Run log: %s", log.path) return 0 if result.crashed_on_install_script: - print( + logger.info( "steamcmd hit the known install-script crash; finishing the manifest manually..." ) else: - print( - f"steamcmd exited {result.returncode} but the manifest doesn't reflect a " - "real install; finishing it manually..." + logger.info( + "steamcmd exited %d but the manifest doesn't reflect a real install; " + "checking whether it is safe to finish it manually...", + result.returncode, ) - manifest_path = recognize.finish_manifest_manually( + return _finish( + log, + library, + metadata, + target, + depot_plan, + expected_build_id, + skipped_steamcmd=args.skip_steamcmd, + ) + + +def _finish( + log: runlog.RunLog, + library: SteamLibrary, + metadata: appinfo.AppMetadata, + target: Path, + depot_plan: appinfo.DepotPlan, + expected_build_id: str | None, + skipped_steamcmd: bool, +) -> int: + """Hand-finish the manifest, but only if every precondition really holds.""" + logger = log.logger + manifest_path = library.steamapps / f"appmanifest_{metadata.appid}.acf" + + if not manifest_path.is_file(): + # Reachable two ways, so don't blame steamcmd unconditionally: under + # --skip-steamcmd it never runs at all, and nothing else in this tool + # creates a manifest from scratch. + logger.error( + "No %s exists, so there is nothing to finish. This tool only ever " + "patches a manifest Steam or steamcmd already wrote; it never " + "fabricates one. %s", + manifest_path, + ( + "Re-run without --skip-steamcmd so steamcmd creates it (it writes " + "one even when it crashes)." + if skipped_steamcmd + else "steamcmd writes one even when it crashes, so its absence " + "means steamcmd never ran far enough." + ), + ) + logger.info("Run log (send this): %s", log.path) + return 1 + + app_state = recognize.read_app_state(manifest_path) + problems = recognize.check_ready_to_finish( + install_dir=target, + app_state=app_state, + required_depots=depot_plan.required, + expected_build_id=expected_build_id, + ) + if problems: + # Fail closed. Writing StateFlags=4 here is what previously turned a + # steamcmd run that did nothing at all into a reported success. + logger.error("Refusing to mark %s as installed:", metadata.name) + for problem in problems: + logger.error(" - %s", problem) + logger.info("Run log (send this): %s", log.path) + return 1 + + # Do this before writing the manifest: a StateFlags=4 manifest alongside a + # staging tree full of pending chunks is the same contradiction that made + # Steam resume the download last time. + freed = recognize.clear_steam_staging(library, metadata.appid) + if freed: + logger.info("Cleared %.1f GB of stale Steam staging data.", freed / 1e9) + + written = recognize.finish_manifest_manually( library=library, appid=metadata.appid, install_dir=target, now_unix=int(time.time()), ) - print(f"Wrote {manifest_path} -- launch Steam and confirm Play works.") + log.file_contents("Manifest after manual finish", written) + + # Re-read from disk: claiming success without checking the write landed is + # the same fail-open mistake in a different place. + if not recognize.manifest_reflects_real_install(library, metadata.appid, target): + logger.error( + "Wrote %s but it still does not read back as a real install.", written + ) + logger.info("Run log (send this): %s", log.path) + return 1 + + logger.info("Wrote %s -- launch Steam and confirm Play works.", written) + logger.info("Run log: %s", log.path) return 0 diff --git a/_installer_import/steam_installer/depotdl.py b/_installer_import/steam_installer/depotdl.py index cc3c60c..09b5cf3 100644 --- a/_installer_import/steam_installer/depotdl.py +++ b/_installer_import/steam_installer/depotdl.py @@ -11,9 +11,11 @@ built to avoid. from __future__ import annotations -import subprocess +import logging from pathlib import Path +from steam_installer import runlog + DEFAULT_MAX_DOWNLOADS = 32 @@ -21,11 +23,17 @@ def download( appid: int, install_dir: Path, username: str, + logger: logging.Logger, language: str = "english", os_name: str = "windows", max_downloads: int = DEFAULT_MAX_DOWNLOADS, -) -> None: - """Run depotdownloader against `install_dir`, streaming its own progress output live.""" +) -> int: + """Run depotdownloader against `install_dir`, streaming its own progress output live. + + Returns the exit code rather than raising: the caller has to distinguish + "download failed" from "download fine, manifest step failed" to decide + whether touching the manifest is safe at all. + """ install_dir.mkdir(parents=True, exist_ok=True) cmd = [ "depotdownloader", @@ -43,4 +51,5 @@ def download( "-dir", str(install_dir), ] - subprocess.run(cmd, check=True) + returncode, _ = runlog.run_logged(cmd, logger) + return returncode diff --git a/_installer_import/steam_installer/recognize.py b/_installer_import/steam_installer/recognize.py index f71c889..59fd033 100644 --- a/_installer_import/steam_installer/recognize.py +++ b/_installer_import/steam_installer/recognize.py @@ -13,25 +13,42 @@ evaluator reports failure (crash backtrace: the same stack frame repeating dozens of times). There's no known fix. When that specific crash is detected, we finish the appmanifest by hand using ONLY values we already verified ourselves: the exact manifest IDs from depotdownloader's own local -cache (never guessed), and the real on-disk size via `du`. Steam is left to -self-correct the rest (byte counters, staged-file bookkeeping) on its very -next launch -- confirmed: it did exactly that in practice, cleaning up the -hand-edited fields on its own. +cache (never guessed), and the real on-disk size measured directly. + +Do NOT leave the download bookkeeping for Steam to tidy up on its next +launch. An earlier version of this module did, on the belief that Steam +self-corrects; it does not. A manifest that says StateFlags=4 while +`BytesDownloaded` still differs from `BytesToDownload` is self-contradictory, +and Steam resolves the contradiction by discarding the "installed" claim and +resuming the download -- 90 GB of it, in the run that prompted this comment. +The byte counters (`_settle_download_counters`) and the staging tree +(`clear_steam_staging`) must both be settled here, and +`manifest_reflects_real_install` refuses to call an install real until they +are. """ from __future__ import annotations +import logging +import os import re +import shutil import signal -import subprocess from dataclasses import dataclass from pathlib import Path -from steam_installer import vdf +from steam_installer import runlog, vdf from steam_installer.library import SteamLibrary MANIFEST_CACHE_RE = re.compile(r"^(\d+)_(\d+)\.manifest$") +# depotdownloader's own cache/staging area lives inside the install dir but is +# not game content -- Steam's SizeOnDisk must not count it. +DEPOTDOWNLOADER_DIR = ".DepotDownloader" + +# Module-level so tests can point it at a fake /proc tree. +PROC_DIR = Path("/proc") + class RecognizeError(Exception): pass @@ -42,31 +59,40 @@ class ValidateResult: completed: bool crashed_on_install_script: bool returncode: int + output: str = "" def run_steamcmd_validate( - appid: int, install_dir: Path, username: str + appid: int, + username: str, + logger: logging.Logger, ) -> ValidateResult: - """Run steamcmd's app_update ... validate. `force_install_dir` must precede `+login` - (steamcmd itself warns about this ordering). + """Run steamcmd's app_update ... validate. - Deliberately does NOT capture stdout/stderr: steamcmd prompts for a - password (and possibly a Steam Guard code) on stdin, and prompt text - written to a captured/buffered stdout never reaches the terminal -- - the process looks hung with no visible request for input. Output is - left to inherit the parent's terminal instead, exactly like - depotdownloader's own interactive login. + Takes no install dir on purpose. `force_install_dir` sets a Steam *library + root*, not an install directory: pointing it at the game folder made + steamcmd look for manifests in `/steamapps/`, find none ("Loaded 0 + apps from install folder"), create that stray directory, and then fail the + app with "Invalid platform" because it had no depot context for it. The app + already lives in a registered library, so letting steamcmd use the real one + is both correct and sufficient. + + Output is teed through a pty (see runlog) rather than captured: steamcmd + prompts for a password and possibly a Steam Guard code on stdin, and prompt + text written to a captured/buffered pipe never reaches the terminal -- the + process looks hung with no visible request for input. The confirmed steamcmd install-script bug (see module docstring) kills the process with SIGSEGV -- Python reports that as returncode -11 on POSIX. Checking the signal directly is more reliable than grepping - captured text for a message string, and doesn't require capturing - output at all. + captured text for a message string. """ cmd = [ "steamcmd", - "+force_install_dir", - str(install_dir), + # A Linux steamcmd refuses a Windows-only app with "Invalid platform" + # unless told which platform to resolve depots for. Must precede +login. + "+@sSteamCmdForcePlatformType", + "windows", "+login", username, "+app_update", @@ -74,12 +100,13 @@ def run_steamcmd_validate( "validate", "+quit", ] - result = subprocess.run(cmd, timeout=3600) - crashed = result.returncode == -signal.SIGSEGV + returncode, output = runlog.run_logged(cmd, logger, timeout=3600) + crashed = returncode == -signal.SIGSEGV return ValidateResult( - completed=result.returncode == 0, + completed=returncode == 0, crashed_on_install_script=crashed, - returncode=result.returncode, + returncode=returncode, + output=output, ) @@ -94,8 +121,8 @@ def ensure_installdir_correct( read that same stale field and wiped the correctly-downloaded folder as "orphaned" while starting a fresh download into the wrong one. Fixing it proactively, before steamcmd ever runs, closes that window entirely. - Does nothing if no manifest exists yet -- steamcmd creates one itself in - that case, with the right installdir since we pass force_install_dir. + Does nothing if no manifest exists yet -- steamcmd's own app_update writes + one in that case, and it derives installdir from the app's metadata. """ manifest_path = library.steamapps / f"appmanifest_{appid}.acf" if not manifest_path.is_file(): @@ -107,7 +134,8 @@ def ensure_installdir_correct( if app_state.get("installdir") == install_dir.name: return app_state["installdir"] = install_dir.name - manifest_path.write_text(vdf.dumps("AppState", app_state)) + _backup_once(manifest_path) + _write_atomic(manifest_path, vdf.dumps("AppState", app_state)) def manifest_reflects_real_install( @@ -132,12 +160,33 @@ def manifest_reflects_real_install( app_state.get("StateFlags") == "4" and app_state.get("installdir") == install_dir.name and bool(app_state.get("InstalledDepots")) + and download_counters_settled(app_state) ) +def download_counters_settled(app_state: dict) -> bool: + """Whether the download bookkeeping agrees that nothing is left to fetch. + + This is the check that was missing when Steam threw away a StateFlags=4 + manifest and resumed a 90 GB download: the file simultaneously claimed + "fully installed" and "22 GB of 92 GB downloaded". Across every genuine + installed manifest on this machine the counters are equal in pairs and + StagingSize is "0", so treat any inequality as "still downloading". + """ + pairs = (("BytesToDownload", "BytesDownloaded"), ("BytesToStage", "BytesStaged")) + for total_key, done_key in pairs: + total = app_state.get(total_key) + done = app_state.get(done_key) + # Absent counters are fine (Steam omits them on some apps); mismatched + # ones are not. + if total is not None and done is not None and total != done: + return False + return app_state.get("StagingSize", "0") == "0" + + def discover_cached_depot_manifests(install_dir: Path) -> dict[str, str]: """Depot ID -> manifest ID, read from depotdownloader's own local cache filenames.""" - cache_dir = install_dir / ".DepotDownloader" + cache_dir = install_dir / DEPOTDOWNLOADER_DIR found: dict[str, str] = {} if not cache_dir.is_dir(): return found @@ -149,6 +198,169 @@ def discover_cached_depot_manifests(install_dir: Path) -> dict[str, str]: return found +def clear_steam_staging(library: SteamLibrary, appid: int) -> int: + """Drop Steam's half-finished staging tree for `appid`. Returns bytes freed. + + Steam stages downloaded chunks in `steamapps/downloading//` and + commits them into `common/` afterwards. If we declare the app installed + while that tree still holds tens of GB plus its `state_*.patch` files, + Steam finds work pending and resumes the update -- the manifest says + "done", the staging area says "mid-download". Every genuinely installed app + on this machine has an empty (4 KB) staging dir, so matching that is what + "installed" actually looks like on disk. + + Only ever discards Steam's own staging copies, never `common/` content. + """ + downloading = library.steamapps / "downloading" + staged = downloading / str(appid) + freed = 0 + if staged.is_dir(): + freed = sum(f.stat().st_size for f in staged.rglob("*") if f.is_file()) + shutil.rmtree(staged) + staged.mkdir(parents=True, exist_ok=True) + for patch in downloading.glob(f"state_{appid}_*.patch"): + freed += patch.stat().st_size + patch.unlink() + return freed + + +def read_app_state(manifest_path: Path) -> dict: + """The `AppState` block of a manifest, or an empty dict if unreadable.""" + if not manifest_path.is_file(): + return {} + data = vdf.parse(manifest_path.read_text()) + app_state = data.get("AppState", {}) + return app_state if isinstance(app_state, dict) else {} + + +def steam_client_pids() -> list[int]: + """PIDs of a running Steam client, if any. + + Steam owns appmanifest_*.acf while it runs and rewrites it from its own + in-memory state, so a manifest patched underneath a live client is simply + discarded -- which is exactly what happened on the run that prompted all + this. Matching on the executable path (not a `pgrep -f` pattern) avoids + matching our own command line. + """ + pids: list[int] = [] + for entry in PROC_DIR.iterdir(): + if not entry.name.isdigit(): + continue + try: + exe = (entry / "exe").resolve() + except OSError: + continue + # Match the client by executable name alone. An earlier version also + # required "ubuntu12_32" in the path, which silently fails open for a + # Flatpak install (/app/bin/steam) -- and a guard that quietly stops + # guarding is worse than no guard. `steamcmd` is a different binary, so + # it is not matched here. + if exe.name in ("steam", "steamwebhelper"): + pids.append(int(entry.name)) + return pids + + +def check_ready_to_finish( + install_dir: Path, + app_state: dict, + required_depots: set[str], + expected_build_id: str | None, +) -> list[str]: + """Reasons the manifest must NOT be hand-finished. Empty list means go ahead. + + Every one of these was a real failure mode: writing StateFlags=4 after a + steamcmd run that did nothing turned a dead run into a reported success. + """ + problems: list[str] = [] + + # An empty required set would make the depot check below vacuously true and + # wave through a download that fetched nothing. It means the depot filter + # matched no depot at all (wrong language or OS for this app), not that + # there is nothing to install. + if not required_depots: + problems.append( + "no depots match this app for the chosen language/OS -- refusing to " + "guess that the install is complete" + ) + + cached = discover_cached_depot_manifests(install_dir) + + # `finish_manifest_manually` drops depots the on-disk manifest lists under + # SharedDepots, which is a different source of truth from app_info's + # depotfromapp/sharedinstall. If they disagree it can end up with nothing to + # install and raise -- and by then clear_steam_staging has already deleted + # Steam's resumable staging tree. Catch the disagreement here, while + # refusing is still free. + shared = app_state.get("SharedDepots") + shared_ids = set(shared) if isinstance(shared, dict) else set() + if cached and not (set(cached) - shared_ids): + problems.append( + "every cached depot is listed under SharedDepots in the existing " + "manifest, so there would be nothing to record as installed" + ) + + missing = sorted(required_depots - set(cached)) + if missing: + problems.append( + f"depot(s) {', '.join(missing)} have no cached manifest under " + f"{install_dir}/{DEPOTDOWNLOADER_DIR} -- the download did not finish" + ) + + buildid = app_state.get("buildid") + target = app_state.get("TargetBuildID") + if not (buildid and buildid != "0") and not (target and target != "0"): + problems.append( + "manifest has no usable buildid (both buildid and TargetBuildID are 0) " + "-- steamcmd never recorded a real build" + ) + resolved = buildid if buildid and buildid != "0" else target + if expected_build_id is None: + # Don't silently skip the comparison: a None here means app_info had no + # branches.public, so we cannot tell a current build from a stale one. + problems.append( + "could not determine the current public build from app info, so the " + "manifest's buildid cannot be checked" + ) + elif resolved and resolved != expected_build_id: + problems.append( + f"manifest buildid {resolved} != current public build " + f"{expected_build_id} -- content is for a different build" + ) + + # Cache *filenames* are not content. Without this, an install dir holding + # only `.DepotDownloader/_.manifest` and no game files passes every + # other check and gets written as StateFlags=4 with SizeOnDisk=0. + if not measure_content_size(install_dir): + problems.append( + f"no game content found under {install_dir} (only depotdownloader " + "cache files) -- nothing was actually installed" + ) + + pids = steam_client_pids() + if pids: + problems.append( + f"the Steam client is running (pid {', '.join(str(p) for p in pids)}) " + "and will overwrite the manifest -- close Steam and re-run" + ) + + return problems + + +def measure_content_size(install_dir: Path) -> int: + """Bytes of real game content, excluding depotdownloader's own cache/staging. + + `.DepotDownloader/` holds manifests and a staging tree that Steam knows + nothing about; counting them overstates SizeOnDisk by gigabytes. + """ + total = 0 + for entry in install_dir.rglob("*"): + if DEPOTDOWNLOADER_DIR in entry.parts: + continue + if entry.is_file() and not entry.is_symlink(): + total += entry.stat().st_size + return total + + def finish_manifest_manually( library: SteamLibrary, appid: int, @@ -175,10 +387,10 @@ def finish_manifest_manually( depot_manifests = discover_cached_depot_manifests(install_dir) if not depot_manifests: raise RecognizeError( - f"No cached depot manifests found under {install_dir}/.DepotDownloader" + f"No cached depot manifests found under {install_dir}/{DEPOTDOWNLOADER_DIR}" ) - size_on_disk = sum(f.stat().st_size for f in install_dir.rglob("*") if f.is_file()) + size_on_disk = measure_content_size(install_dir) # "0" means "no real build installed yet" (steamcmd/Steam use it as a # sentinel) -- a bare `or` treats that non-empty string as truthy and # never falls through to TargetBuildID, leaving a stale/wrong buildid. @@ -186,10 +398,20 @@ def finish_manifest_manually( buildid = existing_buildid if existing_buildid and existing_buildid != "0" else None buildid = buildid or app_state.get("TargetBuildID") or "0" + # A shared depot (e.g. Steamworks Common Redistributables) is installed once + # under its owning app and only referenced here. Listing it in + # InstalledDepots as though this app owned a copy contradicts SharedDepots. + shared = app_state.get("SharedDepots") + shared_ids = set(shared) if isinstance(shared, dict) else set() installed_depots = { depot_id: {"manifest": manifest_id, "size": "0"} for depot_id, manifest_id in depot_manifests.items() + if depot_id not in shared_ids } + if not installed_depots: + raise RecognizeError( + f"Every cached depot for {appid} is a shared depot -- nothing to install" + ) app_state["StateFlags"] = "4" app_state["installdir"] = install_dir.name @@ -200,8 +422,52 @@ def finish_manifest_manually( app_state["UpdateResult"] = "0" app_state["InstalledDepots"] = installed_depots app_state.pop("StagedDepots", None) + _settle_download_counters(app_state) - backup_path = manifest_path.with_suffix(".acf.bak") - backup_path.write_text(manifest_path.read_text()) - manifest_path.write_text(vdf.dumps("AppState", app_state)) + _backup_once(manifest_path) + _write_atomic(manifest_path, vdf.dumps("AppState", app_state)) return manifest_path + + +def _backup_once(manifest_path: Path) -> None: + """Keep the *pristine* manifest, not the previous run's output. + + An unconditional backup means a second run overwrites the original with + run 1's already-patched file, so the recovery artifact for a 90+ GB install + is one re-run away from being useless. Only write it if absent. + """ + backup_path = manifest_path.with_suffix(".acf.bak") + if not backup_path.exists(): + backup_path.write_text(manifest_path.read_text()) + + +def _write_atomic(path: Path, text: str) -> None: + """Write via a temp file + rename so a crash can't truncate the manifest. + + A partial write leaves an unparseable manifest, which Steam and this tool + both then choke on. `os.replace` is atomic within a filesystem, and the + temp file is created alongside the target to stay on the same one. + """ + tmp = path.with_name(f"{path.name}.tmp{os.getpid()}") + try: + tmp.write_text(text) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def _settle_download_counters(app_state: dict) -> None: + """Mark the download bookkeeping complete: `Downloaded` == `ToDownload`. + + Leaving these at their mid-download values is what made Steam discard a + StateFlags=4 manifest and resume fetching. Real installed manifests keep + these fields (they are not dropped) with the pair values equal. + """ + for total_key, done_key in ( + ("BytesToDownload", "BytesDownloaded"), + ("BytesToStage", "BytesStaged"), + ): + total = app_state.get(total_key) + if total is not None: + app_state[done_key] = total diff --git a/_installer_import/steam_installer/runlog.py b/_installer_import/steam_installer/runlog.py new file mode 100644 index 0000000..d3741de --- /dev/null +++ b/_installer_import/steam_installer/runlog.py @@ -0,0 +1,223 @@ +"""Per-run logging. + +Everything a run does lands in one file the user can copy-paste verbatim when +something goes wrong. The first version of this tool had no logging at all, and +when a run failed the evidence was gone: Steam owns appmanifest_.acf and +had already overwritten the manifest we wrote by the time anyone looked. So the +log records the manifest *verbatim, before and after* every write, alongside the +full output of every subprocess. + +Subprocess output is teed through a pty rather than captured with +`capture_output=True`. steamcmd prompts for a password (and possibly a Steam +Guard code) on stdin; prompt text written to a captured/buffered pipe never +reaches the terminal, so the process looks hung with no visible request for +input. A pty keeps steamcmd in line-buffered interactive mode and lets us +mirror every byte to both the terminal and the log. +""" + +from __future__ import annotations + +import errno +import io +import logging +import os +import pty +import select +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +LOG_DIR = Path.home() / ".local/state/steam-game-installer" +LOGGER_NAME = "steam_installer" + +_CONSOLE_FORMAT = "%(message)s" +_FILE_FORMAT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s" + +# Read granularity for the pty relay. Large enough that depotdownloader's +# progress spam doesn't cost a syscall per line, small enough to stay responsive. +_READ_CHUNK = 65536 + + +@dataclass +class RunLog: + """A single run's log file, plus the `latest.log` symlink pointing at it.""" + + path: Path + logger: logging.Logger + + def section(self, title: str) -> None: + """A visual divider so a pasted log is skimmable.""" + self.logger.debug("=" * 72) + self.logger.debug(title) + self.logger.debug("=" * 72) + + def file_contents(self, label: str, path: Path) -> None: + """Record a file verbatim. Used for manifests before/after every write.""" + self.section(f"{label}: {path}") + try: + self.logger.debug("\n%s", path.read_text()) + except OSError as exc: + self.logger.debug("(unreadable: %s)", exc) + + +def setup(appid: int | None, verbose: bool = False) -> RunLog: + """Start a run log. Console stays terse; the file gets everything at DEBUG.""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") + path = LOG_DIR / f"run-{stamp}-{appid if appid is not None else 'noappid'}.log" + + logger = logging.getLogger(LOGGER_NAME) + logger.setLevel(logging.DEBUG) + logger.propagate = False + # A second setup() in the same process (tests) must not double-log. + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + file_handler = logging.FileHandler(path, encoding="utf-8") + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(logging.Formatter(_FILE_FORMAT)) + logger.addHandler(file_handler) + + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.DEBUG if verbose else logging.INFO) + console.setFormatter(logging.Formatter(_CONSOLE_FORMAT)) + logger.addHandler(console) + + _point_latest_at(path) + return RunLog(path=path, logger=logger) + + +def _point_latest_at(path: Path) -> None: + """Best-effort `latest.log` symlink -- a stale symlink must never fail a run.""" + latest = LOG_DIR / "latest.log" + try: + if latest.is_symlink() or latest.exists(): + latest.unlink() + latest.symlink_to(path.name) + except OSError: + pass + + +def run_logged( + cmd: list[str], + logger: logging.Logger, + timeout: float | None = None, +) -> tuple[int, str]: + """Run `cmd` on a pty, mirroring output to both the terminal and the log. + + Returns (returncode, combined_output). stdin stays attached to the real + terminal so interactive prompts (steamcmd's password / Steam Guard) work + exactly as they do without logging. + """ + logger.debug("$ %s", shlex.join(cmd)) + started = time.monotonic() + + master_fd, slave_fd = pty.openpty() + chunks: list[bytes] = [] + try: + process = subprocess.Popen( + cmd, + stdout=slave_fd, + stderr=slave_fd, + stdin=_stdin_for_child(slave_fd), + close_fds=True, + ) + except OSError: + os.close(master_fd) + os.close(slave_fd) + raise + # The child owns the slave end now; holding it open here would keep the + # master readable forever and hang the relay after the child exits. + os.close(slave_fd) + + try: + _relay(master_fd, chunks, process, timeout, started) + except subprocess.TimeoutExpired as exc: + # Record what the child managed to say before it hung, then re-raise. + # Losing it here defeats the whole point of the log in precisely the + # case where it matters most -- a steamcmd hang at the 3600s timeout. + output = _log_output(logger, chunks, started, returncode=None) + process.wait() + exc.output = output.encode() + raise + finally: + os.close(master_fd) + + returncode = process.wait() + output = _log_output(logger, chunks, started, returncode) + return returncode, output + + +def _log_output( + logger: logging.Logger, + chunks: list[bytes], + started: float, + returncode: int | None, +) -> str: + output = b"".join(chunks).decode("utf-8", errors="replace") + elapsed = time.monotonic() - started + logger.debug("--- output (%d bytes) ---\n%s", len(output), output) + if returncode is None: + logger.debug("TIMED OUT after %.1fs", elapsed) + else: + logger.debug("exit=%d elapsed=%.1fs", returncode, elapsed) + return output + + +def _read_pty(fd: int, size: int) -> bytes: + """Indirection so tests can fault-inject the pty read without patching the + global `os.read`, which subprocess itself uses during spawn.""" + return os.read(fd, size) + + +def _stdin_for_child(slave_fd: int) -> int: + """The child's stdin: the real terminal when there is one, else the pty. + + steamcmd needs a readable stdin to prompt for a password, but `sys.stdin` + is not always a real file descriptor -- under pytest's capture, systemd, or + a piped invocation it has no fileno(). Falling back to the pty's slave end + keeps the child on a terminal-like stdin instead of crashing. + """ + try: + return sys.stdin.fileno() + except (AttributeError, ValueError, OSError, io.UnsupportedOperation): + return slave_fd + + +def _relay( + master_fd: int, + chunks: list[bytes], + process: subprocess.Popen, + timeout: float | None, + started: float, +) -> None: + """Pump pty output to stdout and `chunks` until the child closes it.""" + while True: + if timeout is not None and time.monotonic() - started > timeout: + process.kill() + raise subprocess.TimeoutExpired(process.args, timeout) + try: + ready, _, _ = select.select([master_fd], [], [], 0.2) + except OSError: + break + if not ready: + # No output pending; the child may still have exited. + if process.poll() is not None: + break + continue + try: + data = _read_pty(master_fd, _READ_CHUNK) + except OSError as exc: + # EIO is the normal "child closed the pty" signal on Linux. + if exc.errno == errno.EIO: + break + raise + if not data: + break + chunks.append(data) + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() diff --git a/_installer_import/steam_installer/vdf.py b/_installer_import/steam_installer/vdf.py index 11d282f..b73bec3 100644 --- a/_installer_import/steam_installer/vdf.py +++ b/_installer_import/steam_installer/vdf.py @@ -98,6 +98,20 @@ def _tokenize(text: str) -> list[str]: return tokens +def _escape(value: str) -> str: + r"""Re-escape a value for VDF output. + + The tokenizer consumes `\` escapes, so a manifest holding + `_CommonRedist\\vcredist\\2015\\installscript.vdf` parses to a value with + single backslashes. Writing that back verbatim would halve them on every + parse/dump cycle and silently corrupt the path -- confirmed against the + real appmanifest_228980.acf (Steamworks Common Redistributables), whose + InstallScripts entries are Windows paths and whose depot 228989 is shared + with other games. + """ + return value.replace("\\", "\\\\").replace('"', '\\"') + + def dumps(root_key: str, data: dict[str, VdfValue], indent: str = "\t") -> str: """Serialize a dict back to VDF text, matching Steam's own tab-indented style.""" lines = [f'"{root_key}"', "{"] @@ -112,9 +126,9 @@ def _dump_block( prefix = indent * depth for key, value in data.items(): if isinstance(value, dict): - lines.append(f'{prefix}"{key}"') + lines.append(f'{prefix}"{_escape(key)}"') lines.append(f"{prefix}{{") _dump_block(value, indent, depth + 1, lines) lines.append(f"{prefix}}}") else: - lines.append(f'{prefix}"{key}"\t\t"{value}"') + lines.append(f'{prefix}"{_escape(key)}"\t\t"{_escape(value)}"') diff --git a/_installer_import/tests/conftest.py b/_installer_import/tests/conftest.py new file mode 100644 index 0000000..fbc4176 --- /dev/null +++ b/_installer_import/tests/conftest.py @@ -0,0 +1,21 @@ +"""Shared test setup. + +`test_module_is_runnable_as_a_script` runs `python -m steam_installer.cli` in a +subprocess. Without the hook below, coverage collected in that child is thrown +away and the `__main__` guard looks uncovered. +""" + +import os +import pathlib + +import pytest + + +@pytest.fixture +def coverage_subprocess_env(): + """Env for a child process that should also record coverage.""" + env = os.environ.copy() + root = pathlib.Path(__file__).resolve().parent.parent + env["COVERAGE_PROCESS_START"] = str(root / "pyproject.toml") + env["PYTHONPATH"] = str(root) + os.pathsep + env.get("PYTHONPATH", "") + return env diff --git a/_installer_import/tests/test_appinfo.py b/_installer_import/tests/test_appinfo.py new file mode 100644 index 0000000..06ab228 --- /dev/null +++ b/_installer_import/tests/test_appinfo.py @@ -0,0 +1,236 @@ +import pytest + +from steam_installer import appinfo + + +def _app_block(monkeypatch, block): + monkeypatch.setattr(appinfo, "_fetch_app_block", lambda appid: block) + + +# Shape taken from a real `app_info_print 1771300`: 12 depots, of which an +# english/windows install only ever fetches three. +KCD2_DEPOTS = { + "228989": {"depotfromapp": "228980", "config": {"oslist": "windows"}}, + "1771302": {"config": {"oslist": "windows"}}, + "1771303": {"config": {"oslist": "windows"}}, + "1771304": {"dlcappid": "3406020", "config": {"oslist": "windows"}}, + "1771305": {"config": {"oslist": "windows", "language": "czech"}}, + "1771306": {"config": {"oslist": "windows", "language": "english"}}, + "1771307": {"config": {"oslist": "windows", "language": "french"}}, + "4075290": {"dlcappid": "4075290"}, + "branches": {"public": {"buildid": "23914554"}}, +} + + +def test_get_depot_plan_selects_only_this_language_and_os(monkeypatch): + _app_block(monkeypatch, {"depots": KCD2_DEPOTS}) + + plan = appinfo.get_depot_plan(1771300, language="english", os_name="windows") + + assert plan.required == {"1771302", "1771303", "1771306"} + assert plan.shared == {"228989"} + assert plan.all_expected == {"1771302", "1771303", "1771306", "228989"} + + +def test_get_depot_plan_switches_language(monkeypatch): + _app_block(monkeypatch, {"depots": KCD2_DEPOTS}) + plan = appinfo.get_depot_plan(1771300, language="czech") + assert "1771305" in plan.required + assert "1771306" not in plan.required + + +def test_get_depot_plan_excludes_other_os(monkeypatch): + _app_block( + monkeypatch, + {"depots": {"10": {"config": {"oslist": "linux"}}, "11": {"config": {}}}}, + ) + plan = appinfo.get_depot_plan(1771300, os_name="windows") + assert plan.required == {"11"} + + +def test_get_depot_plan_skips_optional_depots(monkeypatch): + _app_block(monkeypatch, {"depots": {"10": {"optional": "1"}, "11": {}}}) + assert appinfo.get_depot_plan(1771300).required == {"11"} + + +def test_get_depot_plan_rejects_malformed_depots_section(monkeypatch): + _app_block(monkeypatch, {"depots": "not-a-dict"}) + with pytest.raises(appinfo.AppInfoError, match="depots"): + appinfo.get_depot_plan(1771300) + + +def test_get_depot_plan_with_no_depots_is_empty(monkeypatch): + """An app with no depot list yields nothing required -- callers must not + read that as "everything downloaded fine".""" + _app_block(monkeypatch, {}) + plan = appinfo.get_depot_plan(1771300) + assert plan.required == set() + assert plan.shared == set() + + +def test_get_build_id_reads_public_branch(monkeypatch): + _app_block(monkeypatch, {"depots": KCD2_DEPOTS}) + assert appinfo.get_build_id(1771300) == "23914554" + + +def test_get_build_id_missing_branch_returns_none(monkeypatch): + _app_block(monkeypatch, {"depots": {"branches": {}}}) + assert appinfo.get_build_id(1771300) is None + + +def test_get_build_id_without_depots_returns_none(monkeypatch): + _app_block(monkeypatch, {}) + assert appinfo.get_build_id(1771300) is None + + +def test_get_metadata_reads_name_and_installdir(monkeypatch): + _app_block( + monkeypatch, + { + "common": { + "name": "Kingdom Come: Deliverance II", + "installdir": "KingdomComeDeliverance2", + } + }, + ) + metadata = appinfo.get_metadata(1771300) + assert metadata.name == "Kingdom Come: Deliverance II" + assert metadata.installdir == "KingdomComeDeliverance2" + + +def test_get_metadata_falls_back_to_config_installdir(monkeypatch): + _app_block( + monkeypatch, + {"common": {"name": "Game"}, "config": {"installdir": "GameDir"}}, + ) + assert appinfo.get_metadata(1).installdir == "GameDir" + + +def test_get_metadata_requires_name(monkeypatch): + _app_block(monkeypatch, {"common": {"installdir": "GameDir"}}) + with pytest.raises(appinfo.AppInfoError, match="no name field"): + appinfo.get_metadata(1) + + +def test_get_metadata_requires_installdir(monkeypatch): + _app_block(monkeypatch, {"common": {"name": "Game"}}) + with pytest.raises(appinfo.AppInfoError, match="no installdir"): + appinfo.get_metadata(1) + + +def test_get_metadata_rejects_malformed_common_section(monkeypatch): + _app_block(monkeypatch, {"common": "not-a-dict"}) + with pytest.raises(appinfo.AppInfoError, match="no 'common' section"): + appinfo.get_metadata(1) + + +class _FakeCompleted: + def __init__(self, stdout): + self.stdout = stdout + self.returncode = 0 + + +# What steamcmd actually emits: its own log lines, then the VDF block. +STEAMCMD_OUTPUT = """Redirecting stderr to '/home/u/.steam/logs/stderr.txt' +Logging directory: '/home/u/.steam/logs' +"@sSteamCmdForcePlatformType" = "windows" +Logging in user 'anonymous' to Steam Public...OK +"1771300" +{ +\t"common" +\t{ +\t\t"name"\t\t"Kingdom Come: Deliverance II" +\t\t"installdir"\t\t"KingdomComeDeliverance2" +\t} +\t"depots" +\t{ +\t\t"1771303" +\t\t{ +\t\t\t"config" +\t\t\t{ +\t\t\t\t"oslist"\t\t"windows" +\t\t\t} +\t\t} +\t\t"branches" +\t\t{ +\t\t\t"public" +\t\t\t{ +\t\t\t\t"buildid"\t\t"23914554" +\t\t\t} +\t\t} +\t} +} +""" + + +def test_fetch_app_block_parses_real_steamcmd_output(monkeypatch): + seen = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + return _FakeCompleted(STEAMCMD_OUTPUT) + + monkeypatch.setattr(appinfo.subprocess, "run", fake_run) + + metadata = appinfo.get_metadata(1771300) + + assert metadata.name == "Kingdom Come: Deliverance II" + assert metadata.installdir == "KingdomComeDeliverance2" + # The platform flag must precede +login or a Windows-only app comes back empty. + cmd = seen["cmd"] + assert cmd.index("+@sSteamCmdForcePlatformType") < cmd.index("+login") + assert cmd[cmd.index("+@sSteamCmdForcePlatformType") + 1] == "windows" + + +def test_get_build_id_through_the_real_parser(monkeypatch): + monkeypatch.setattr( + appinfo.subprocess, "run", lambda cmd, **kw: _FakeCompleted(STEAMCMD_OUTPUT) + ) + assert appinfo.get_build_id(1771300) == "23914554" + + +def test_fetch_app_block_raises_when_appid_header_absent(monkeypatch): + """steamcmd prints its usual log noise even for an unknown app; without the + `""` header there is no app info to parse.""" + monkeypatch.setattr( + appinfo.subprocess, + "run", + lambda cmd, **kw: _FakeCompleted("Logging in user 'anonymous'...OK\n"), + ) + with pytest.raises(appinfo.AppInfoError, match="did not return app info"): + appinfo.get_metadata(999999999) + + +def test_fetch_app_block_raises_when_block_is_not_a_mapping(monkeypatch): + """Defensive guard against a shape vdf.parse cannot currently produce (it + always yields a dict for a `{ ... }` block). Exercised by injecting a parser + that returns a scalar, so the guard stays correct if vdf ever changes.""" + monkeypatch.setattr( + appinfo.subprocess, + "run", + lambda cmd, **kw: _FakeCompleted('"1771300"\n{\n}\n'), + ) + monkeypatch.setattr(appinfo.vdf, "parse", lambda text: {"1771300": "scalar"}) + + with pytest.raises(appinfo.AppInfoError, match="unexpected app_info_print"): + appinfo.get_metadata(1771300) + + +def test_get_build_id_handles_non_dict_sections(monkeypatch): + for text in ( + '"1"\n{\n\t"depots"\t\t"scalar"\n}\n', + '"1"\n{\n\t"depots"\n\t{\n\t\t"branches"\t\t"scalar"\n\t}\n}\n', + '"1"\n{\n\t"depots"\n\t{\n\t\t"branches"\n\t\t{\n\t\t\t"public"\t\t"x"\n\t\t}\n\t}\n}\n', + ): + monkeypatch.setattr( + appinfo.subprocess, "run", lambda cmd, _t=text, **kw: _FakeCompleted(_t) + ) + assert appinfo.get_build_id(1) is None + + +def test_get_build_id_returns_none_when_buildid_not_a_string(monkeypatch): + text = '"1"\n{\n\t"depots"\n\t{\n\t\t"branches"\n\t\t{\n\t\t\t"public"\n\t\t\t{\n\t\t\t\t"buildid"\n\t\t\t\t{\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n' + monkeypatch.setattr( + appinfo.subprocess, "run", lambda cmd, **kw: _FakeCompleted(text) + ) + assert appinfo.get_build_id(1) is None diff --git a/_installer_import/tests/test_cli.py b/_installer_import/tests/test_cli.py new file mode 100644 index 0000000..0170056 --- /dev/null +++ b/_installer_import/tests/test_cli.py @@ -0,0 +1,328 @@ +"""End-to-end-ish tests for the CLI's decision logic. + +The bug that motivated all of this lived here, not in the manifest writer: +steamcmd died instantly having done nothing, and main() still wrote +StateFlags=4, printed "launch Steam and confirm Play works" and returned 0. +These tests pin the fail-closed behaviour. +""" + +import logging +import subprocess +import sys + +import pytest + +from steam_installer import appinfo, cli, recognize, resolve, runlog, vdf +from steam_installer.library import SteamLibrary + + +@pytest.fixture(autouse=True) +def isolated_logs(tmp_path, monkeypatch): + monkeypatch.setattr(runlog, "LOG_DIR", tmp_path / "state") + yield + logger = logging.getLogger(runlog.LOGGER_NAME) + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + +@pytest.fixture +def steam(tmp_path, monkeypatch): + """A Steam tree with KCD2 downloaded but its manifest not yet finished.""" + root = tmp_path / "Steam" + library = SteamLibrary(root) + install_dir = library.common / "KingdomComeDeliverance2" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + library.steamapps.mkdir(parents=True, exist_ok=True) + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps( + "AppState", + { + "appid": "1771300", + "StateFlags": "1026", + "buildid": "23914554", + "BytesToDownload": "92009052960", + "BytesDownloaded": "22129058224", + }, + ) + ) + + monkeypatch.setattr(cli, "find_steam_root", lambda: root) + monkeypatch.setattr(cli, "list_libraries", lambda r: [library]) + monkeypatch.setattr(cli, "pick_install_target", lambda libs, d: install_dir) + monkeypatch.setattr(cli.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr( + cli.appinfo, + "get_metadata", + lambda appid: appinfo.AppMetadata(appid, "KCD2", "KingdomComeDeliverance2"), + ) + monkeypatch.setattr( + cli.appinfo, + "get_depot_plan", + lambda appid, language: appinfo.DepotPlan({"1771303"}, {"228989"}), + ) + monkeypatch.setattr(cli.appinfo, "get_build_id", lambda appid: "23914554") + monkeypatch.setattr(recognize, "steam_client_pids", lambda: []) + monkeypatch.setattr(cli.recognize, "steam_client_pids", lambda: []) + monkeypatch.setattr("builtins.input", lambda *a: "y") + return library, install_dir + + +ARGV = ["--appid", "1771300", "--username", "u", "--language", "english"] + + +def _stub_download(monkeypatch, returncode=0): + monkeypatch.setattr(cli.depotdl, "download", lambda **kw: returncode) + + +def _stub_steamcmd(monkeypatch, returncode, crashed=False): + monkeypatch.setattr( + cli.recognize, + "run_steamcmd_validate", + lambda appid, username, logger: recognize.ValidateResult( + completed=returncode == 0, + crashed_on_install_script=crashed, + returncode=returncode, + ), + ) + + +def test_dead_steamcmd_run_still_finishes_when_content_is_present(steam, monkeypatch): + """steamcmd failing is survivable -- the content is what matters -- but the + manifest must come out consistent, not merely flagged installed.""" + library, install_dir = steam + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + + assert cli.main(ARGV) == 0 + + app_state = recognize.read_app_state(library.steamapps / "appmanifest_1771300.acf") + assert app_state["StateFlags"] == "4" + assert app_state["BytesDownloaded"] == app_state["BytesToDownload"] + assert recognize.download_counters_settled(app_state) + + +def test_refuses_when_a_required_depot_never_downloaded(steam, monkeypatch): + """The exact fail-open case: nothing was really downloaded, so declaring the + app installed would be a fabricated success.""" + library, install_dir = steam + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + monkeypatch.setattr( + cli.appinfo, + "get_depot_plan", + lambda appid, language: appinfo.DepotPlan({"1771303", "9999999"}, set()), + ) + + assert cli.main(ARGV) == 1 + + app_state = recognize.read_app_state(library.steamapps / "appmanifest_1771300.acf") + assert app_state["StateFlags"] == "1026" + + +def test_refuses_when_depotdownloader_fails(steam, monkeypatch): + library, _ = steam + _stub_download(monkeypatch, returncode=1) + _stub_steamcmd(monkeypatch, returncode=0) + + assert cli.main(ARGV) == 1 + app_state = recognize.read_app_state(library.steamapps / "appmanifest_1771300.acf") + assert app_state["StateFlags"] == "1026" + + +def test_refuses_before_downloading_when_steam_is_running(steam, monkeypatch): + """Steam owns the manifest while it runs; refuse up front rather than after + an hour of downloading.""" + library, _ = steam + monkeypatch.setattr(cli.recognize, "steam_client_pids", lambda: [4242]) + called = [] + monkeypatch.setattr(cli.depotdl, "download", lambda **kw: called.append(1) or 0) + + assert cli.main(ARGV) == 1 + assert called == [] + + +def test_clean_steamcmd_run_short_circuits(steam, monkeypatch): + """If steamcmd genuinely finished the job, don't hand-patch anything.""" + library, install_dir = steam + _stub_download(monkeypatch) + + def fake_validate(appid, username, logger): + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps( + "AppState", + { + "StateFlags": "4", + "installdir": "KingdomComeDeliverance2", + "InstalledDepots": {"1771303": {"manifest": "1"}}, + "BytesToDownload": "1", + "BytesDownloaded": "1", + }, + ) + ) + return recognize.ValidateResult(True, False, 0) + + monkeypatch.setattr(cli.recognize, "run_steamcmd_validate", fake_validate) + called = [] + monkeypatch.setattr( + cli.recognize, + "finish_manifest_manually", + lambda **kw: called.append(1), + ) + + assert cli.main(ARGV) == 0 + assert called == [] + + +def test_user_declining_confirmation_aborts(steam, monkeypatch): + monkeypatch.setattr("builtins.input", lambda *a: "n") + called = [] + monkeypatch.setattr(cli.depotdl, "download", lambda **kw: called.append(1) or 0) + assert cli.main(ARGV) == 1 + assert called == [] + + +def test_unhandled_error_returns_nonzero_and_names_the_log(steam, monkeypatch, capsys): + def boom(**kwargs): + raise RuntimeError("depotdownloader exploded") + + monkeypatch.setattr(cli.depotdl, "download", boom) + + assert cli.main(ARGV) == 1 + assert "Run log" in capsys.readouterr().out + + +def test_requires_a_name_or_appid(): + with pytest.raises(SystemExit): + cli.main(["--username", "u"]) + + +def test_missing_manifest_under_skip_steamcmd_does_not_blame_steamcmd( + steam, monkeypatch, capsys +): + """--skip-steamcmd never runs steamcmd, so "steamcmd never ran far enough" + would send the user chasing the wrong thing.""" + library, _ = steam + (library.steamapps / "appmanifest_1771300.acf").unlink() + _stub_download(monkeypatch) + + assert cli.main([*ARGV, "--skip-steamcmd"]) == 1 + + out = capsys.readouterr().out + assert "never fabricates one" in out + assert "Re-run without --skip-steamcmd" in out + assert "steamcmd never ran far enough" not in out + + +def test_missing_manifest_with_steamcmd_blames_steamcmd(steam, monkeypatch, capsys): + library, _ = steam + (library.steamapps / "appmanifest_1771300.acf").unlink() + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + + assert cli.main(ARGV) == 1 + assert "steamcmd never ran far enough" in capsys.readouterr().out + + +def test_require_tool_exits_when_binary_missing(monkeypatch, capsys): + monkeypatch.setattr(cli.shutil, "which", lambda name: None) + with pytest.raises(SystemExit): + cli._require_tool("depotdownloader", "See ~/depotdownloader.") + assert "not found on PATH" in capsys.readouterr().err + + +def test_resolve_appid_uses_search_when_no_appid(monkeypatch): + monkeypatch.setattr( + cli.resolve, "search", lambda name: [resolve.SearchResult(1771300, "KCD2")] + ) + monkeypatch.setattr(cli.resolve, "prompt_confirm", lambda results: results[0]) + + args = cli._build_parser().parse_args(["kingdom", "--username", "u"]) + assert cli._resolve_appid(args) == 1771300 + + +def test_resolve_appid_exits_when_search_unavailable(monkeypatch, capsys): + """store.steampowered.com blocks scripted clients on some networks; tell the + user to pass --appid rather than dying with a traceback.""" + + def boom(name): + raise resolve.SearchUnavailable("tls blocked") + + monkeypatch.setattr(cli.resolve, "search", boom) + args = cli._build_parser().parse_args(["kingdom", "--username", "u"]) + + with pytest.raises(SystemExit): + cli._resolve_appid(args) + assert "pass --appid" in capsys.readouterr().err + + +def test_language_defaults_to_english_when_prompt_is_empty(steam, monkeypatch): + seen = {} + monkeypatch.setattr("builtins.input", lambda *a: "y" if "Proceed" in a[0] else "") + monkeypatch.setattr( + cli.appinfo, + "get_depot_plan", + lambda appid, language: ( + seen.setdefault("language", language) + and None + or appinfo.DepotPlan({"1771303"}, set()) + ), + ) + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + + cli.main(["--appid", "1771300", "--username", "u"]) + + assert seen["language"] == "english" + + +def test_reports_the_known_install_script_crash(steam, monkeypatch, capsys): + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=-11, crashed=True) + + assert cli.main(ARGV) == 0 + assert "known install-script crash" in capsys.readouterr().out + + +def test_reports_cleared_staging(steam, monkeypatch, capsys): + library, _ = steam + staged = library.steamapps / "downloading" / "1771300" + staged.mkdir(parents=True) + (staged / "big.pak").write_bytes(b"x" * 2_000_000) + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + + assert cli.main(ARGV) == 0 + assert "stale Steam staging data" in capsys.readouterr().out + + +def test_fails_when_written_manifest_does_not_read_back_as_installed( + steam, monkeypatch, capsys +): + """Claiming success without re-reading the write is the same fail-open + mistake in a different place.""" + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + monkeypatch.setattr( + cli.recognize, "manifest_reflects_real_install", lambda *a, **kw: False + ) + + assert cli.main(ARGV) == 1 + assert "does not read back as a real install" in capsys.readouterr().out + + +def test_module_is_runnable_as_a_script(coverage_subprocess_env): + """`python -m steam_installer.cli` is a real entry point (the __main__ + guard), so it must not be dead code.""" + result = subprocess.run( + [sys.executable, "-m", "steam_installer.cli", "--help"], + capture_output=True, + text=True, + timeout=60, + env=coverage_subprocess_env, + ) + assert result.returncode == 0 + assert "--skip-steamcmd" in result.stdout diff --git a/_installer_import/tests/test_depotdl.py b/_installer_import/tests/test_depotdl.py new file mode 100644 index 0000000..f43cd4e --- /dev/null +++ b/_installer_import/tests/test_depotdl.py @@ -0,0 +1,80 @@ +import logging + +from steam_installer import depotdl + + +def _capture(monkeypatch): + """Intercept the subprocess call and hand back the argv it was given.""" + seen = {} + + def fake_run_logged(cmd, logger, timeout=None): + seen["cmd"] = cmd + return seen.get("rc", 0), "" + + monkeypatch.setattr(depotdl.runlog, "run_logged", fake_run_logged) + return seen + + +def test_download_builds_expected_argv(tmp_path, monkeypatch): + seen = _capture(monkeypatch) + + depotdl.download( + appid=1771300, + install_dir=tmp_path / "Game", + username="krzys2000", + logger=logging.getLogger("t"), + language="english", + max_downloads=32, + ) + + cmd = seen["cmd"] + assert cmd[0] == "depotdownloader" + pairs = dict(zip(cmd[1::2], cmd[2::2])) + assert pairs["-app"] == "1771300" + assert pairs["-language"] == "english" + assert pairs["-max-downloads"] == "32" + assert pairs["-username"] == "krzys2000" + + +def test_download_always_passes_os_windows(tmp_path, monkeypatch): + """Without -os, depotdownloader defaults to the host OS and can filter out + every real (Windows-tagged) depot -- the original "0 bytes from 0 depots".""" + seen = _capture(monkeypatch) + + depotdl.download( + appid=1771300, + install_dir=tmp_path / "Game", + username="u", + logger=logging.getLogger("t"), + ) + + cmd = seen["cmd"] + assert cmd[cmd.index("-os") + 1] == "windows" + assert "-remember-password" in cmd + + +def test_download_creates_the_install_dir(tmp_path, monkeypatch): + _capture(monkeypatch) + target = tmp_path / "nested" / "Game" + + depotdl.download( + appid=1, install_dir=target, username="u", logger=logging.getLogger("t") + ) + + assert target.is_dir() + + +def test_download_returns_the_exit_code_instead_of_raising(tmp_path, monkeypatch): + """The caller must distinguish "download failed" from "manifest step failed" + to decide whether touching the manifest is safe at all.""" + seen = _capture(monkeypatch) + seen["rc"] = 1 + + returncode = depotdl.download( + appid=1, + install_dir=tmp_path / "Game", + username="u", + logger=logging.getLogger("t"), + ) + + assert returncode == 1 diff --git a/_installer_import/tests/test_library.py b/_installer_import/tests/test_library.py index e6ec1c0..a6f2d15 100644 --- a/_installer_import/tests/test_library.py +++ b/_installer_import/tests/test_library.py @@ -1,5 +1,8 @@ from pathlib import Path +import pytest + +from steam_installer import library, vdf from steam_installer.library import ( SteamLibrary, find_existing_install, @@ -68,3 +71,86 @@ def test_pick_install_target_defaults_to_primary_library(tmp_path): libraries = [SteamLibrary(root)] target = pick_install_target(libraries, "NewGame") assert target == root / "steamapps" / "common" / "NewGame" + + +def test_find_steam_root_prefers_default_location(tmp_path, monkeypatch): + root = tmp_path / ".local/share/Steam" + (root / "steamapps").mkdir(parents=True) + monkeypatch.setattr(library, "DEFAULT_STEAM_ROOT", root) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + + assert library.find_steam_root() == root + + +def test_find_steam_root_falls_back_to_dot_steam(tmp_path, monkeypatch): + fallback = tmp_path / ".steam/steam" + (fallback / "steamapps").mkdir(parents=True) + monkeypatch.setattr(library, "DEFAULT_STEAM_ROOT", tmp_path / "absent") + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + + assert library.find_steam_root() == fallback + + +def test_find_steam_root_raises_when_no_steam_install(tmp_path, monkeypatch): + monkeypatch.setattr(library, "DEFAULT_STEAM_ROOT", tmp_path / "absent") + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + + with pytest.raises(FileNotFoundError, match="No Steam installation"): + library.find_steam_root() + + +def test_list_libraries_ignores_malformed_libraryfolders(tmp_path): + """A malformed or partial libraryfolders.vdf must degrade to "just the + primary library", never crash the run.""" + root = tmp_path / "Steam" + (root / "steamapps").mkdir(parents=True) + (root / "steamapps" / "libraryfolders.vdf").write_text( + vdf.dumps("libraryfolders", {"contentstatsid": "123"}) + ) + + assert [lib.path for lib in library.list_libraries(root)] == [root] + + +def test_list_libraries_skips_entries_without_a_path(tmp_path): + root = tmp_path / "Steam" + (root / "steamapps").mkdir(parents=True) + other = tmp_path / "hdd" + other.mkdir() + (root / "steamapps" / "libraryfolders.vdf").write_text( + vdf.dumps( + "libraryfolders", + { + "0": {"path": str(root)}, + "1": {"label": "no path here"}, + "2": {"path": str(other)}, + }, + ) + ) + + paths = [lib.path for lib in library.list_libraries(root)] + assert paths == [root, other.resolve()] + + +def test_list_libraries_ignores_non_dict_folder_entries(tmp_path): + root = tmp_path / "Steam" + (root / "steamapps").mkdir(parents=True) + (root / "steamapps" / "libraryfolders.vdf").write_text( + '"libraryfolders"\n{\n\t"0"\t\t"not-a-dict"\n}\n' + ) + + assert [lib.path for lib in library.list_libraries(root)] == [root] + + +def test_list_libraries_ignores_non_dict_libraryfolders_root(tmp_path, monkeypatch): + """Defensive guard: vdf.parse always yields a dict for a block today, so + inject a parser that does not, keeping the guard honest if vdf changes.""" + root = tmp_path / "Steam" + (root / "steamapps").mkdir(parents=True) + (root / "steamapps" / "libraryfolders.vdf").write_text( + vdf.dumps("libraryfolders", {"0": {"path": str(tmp_path)}}) + ) + monkeypatch.setattr( + "steam_installer.library.vdf.parse", lambda text: {"libraryfolders": "scalar"} + ) + + assert [lib.path for lib in library.list_libraries(root)] == [root] diff --git a/_installer_import/tests/test_recognize.py b/_installer_import/tests/test_recognize.py index 6a318cf..8a9f258 100644 --- a/_installer_import/tests/test_recognize.py +++ b/_installer_import/tests/test_recognize.py @@ -1,15 +1,24 @@ +import logging +import signal from pathlib import Path import pytest -from steam_installer import vdf +from steam_installer import recognize, vdf from steam_installer.library import SteamLibrary from steam_installer.recognize import ( RecognizeError, + check_ready_to_finish, + clear_steam_staging, discover_cached_depot_manifests, + download_counters_settled, ensure_installdir_correct, finish_manifest_manually, manifest_reflects_real_install, + measure_content_size, + read_app_state, + run_steamcmd_validate, + steam_client_pids, ) @@ -214,3 +223,543 @@ def test_manifest_reflects_real_install_false_when_no_manifest(tmp_path): library = SteamLibrary(tmp_path / "Steam") library.steamapps.mkdir(parents=True) assert not manifest_reflects_real_install(library, 1771300, library.common / "Game") + + +def test_finish_manifest_settles_download_counters(tmp_path): + """The regression that started all this: a manifest claiming StateFlags=4 + while still saying 22 GB of 92 GB downloaded. Steam discarded the "installed" + claim and resumed a 90 GB download.""" + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + data = { + "appid": "1771300", + "StateFlags": "1026", + "buildid": "23914554", + "DownloadType": "1", + "BytesToDownload": "92034161488", + "BytesDownloaded": "22129058224", + "BytesToStage": "96447764586", + "BytesStaged": "22462334559", + "StagingSize": "25876914424", + } + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data) + ) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert app_state["BytesDownloaded"] == app_state["BytesToDownload"] + assert app_state["BytesStaged"] == app_state["BytesToStage"] + assert app_state["StagingSize"] == "0" + # DownloadType is present on every real installed manifest -- keep it. + assert app_state["DownloadType"] == "1" + assert download_counters_settled(app_state) + + +def test_finish_manifest_excludes_shared_depots_from_installed(tmp_path): + """228989 is a SharedDepots entry owned by app 228980; claiming this app + installed its own copy contradicts SharedDepots.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "228989_5753583882400741046.manifest").write_bytes(b"x") + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert set(app_state["InstalledDepots"]) == {"1771303"} + assert app_state["SharedDepots"] == {"228989": "228980"} + + +def test_finish_manifest_rejects_only_shared_depots(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "228989_5753583882400741046.manifest").write_bytes(b"x") + + with pytest.raises(RecognizeError, match="shared depot"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + +def test_measure_content_size_excludes_depotdownloader_cache(tmp_path): + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" / "staging" / "Data" + cache_dir.mkdir(parents=True) + (install_dir / "game.bin").write_bytes(b"x" * 100) + (install_dir / ".DepotDownloader" / "d.manifest").write_bytes(b"y" * 5000) + (cache_dir / "staged.pak").write_bytes(b"z" * 9000) + + assert measure_content_size(install_dir) == 100 + + +def test_download_counters_settled_detects_partial_download(): + assert not download_counters_settled( + {"BytesToDownload": "92", "BytesDownloaded": "22"} + ) + assert not download_counters_settled({"BytesToStage": "96", "BytesStaged": "22"}) + assert not download_counters_settled({"StagingSize": "25876914424"}) + assert download_counters_settled( + {"BytesToDownload": "92", "BytesDownloaded": "92", "StagingSize": "0"} + ) + # Steam omits these on some apps; absence must not be treated as a failure. + assert download_counters_settled({}) + + +def test_manifest_reflects_real_install_false_when_counters_unsettled(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + data = { + "StateFlags": "4", + "installdir": "Game", + "InstalledDepots": {"1771303": {"manifest": "1999516201078544527"}}, + "BytesToDownload": "92034161488", + "BytesDownloaded": "22129058224", + } + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data) + ) + assert not manifest_reflects_real_install(library, 1771300, library.common / "Game") + + +def test_check_ready_to_finish_blocks_missing_depot(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771302", "1771303"}, + expected_build_id="23914554", + ) + assert any("1771303" in p for p in problems) + + +def test_check_ready_to_finish_blocks_zero_buildid(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "0", "TargetBuildID": "0"}, + required_depots={"1771303"}, + expected_build_id=None, + ) + assert any("buildid" in p for p in problems) + + +def test_check_ready_to_finish_blocks_build_mismatch(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "11111111"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + assert any("different build" in p for p in problems) + + +def test_check_ready_to_finish_blocks_running_steam(tmp_path, monkeypatch): + """Steam owns the manifest while it runs and rewrites it from its own state.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: [4242]) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + assert any("4242" in p for p in problems) + + +def test_check_ready_to_finish_passes_when_everything_holds(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + + assert ( + check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + == [] + ) + + +def test_read_app_state_missing_file(tmp_path): + assert read_app_state(tmp_path / "nope.acf") == {} + + +def test_clear_steam_staging_removes_tree_and_patches(tmp_path): + """A StateFlags=4 manifest next to a full staging tree is the same + contradiction that made Steam resume a 90 GB download.""" + library = SteamLibrary(tmp_path / "Steam") + downloading = library.steamapps / "downloading" + staged = downloading / "1771300" + (staged / "Data").mkdir(parents=True) + (staged / "Data" / "big.pak").write_bytes(b"x" * 500) + (downloading / "state_1771300_1771303.patch").write_bytes(b"y" * 50) + # Another app's staging must be left completely alone. + other = downloading / "1628350" + other.mkdir(parents=True) + (other / "keep.pak").write_bytes(b"z" * 10) + (downloading / "state_1628350_1.patch").write_bytes(b"z" * 10) + + freed = clear_steam_staging(library, 1771300) + + assert freed == 550 + assert staged.is_dir() + assert list(staged.iterdir()) == [] + assert not (downloading / "state_1771300_1771303.patch").exists() + assert (other / "keep.pak").is_file() + assert (downloading / "state_1628350_1.patch").is_file() + + +def test_clear_steam_staging_when_nothing_staged(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + (library.steamapps / "downloading").mkdir(parents=True) + assert clear_steam_staging(library, 1771300) == 0 + + +def test_check_ready_to_finish_blocks_empty_required_depot_set(tmp_path, monkeypatch): + """An empty required set would make the depot check vacuously true and wave + through a download that fetched nothing.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + (install_dir / ".DepotDownloader").mkdir(parents=True) + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots=set(), + expected_build_id="23914554", + ) + assert any("no depots match" in p for p in problems) + + +def test_backup_keeps_the_pristine_manifest_across_reruns(tmp_path): + """A second run must not overwrite the backup with run 1's patched output -- + that turns the recovery artifact for a 90+ GB install into a copy of the + thing it is supposed to protect against.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + manifest = library.steamapps / "appmanifest_1771300.acf" + pristine = manifest.read_text() + install_dir = library.common / "Game" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + finish_manifest_manually(library, 1771300, install_dir, now_unix=2000) + + assert manifest.with_suffix(".acf.bak").read_text() == pristine + + +def test_manifest_write_is_atomic(tmp_path, monkeypatch): + """A crash mid-write must not leave a truncated, unparseable manifest.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + manifest = library.steamapps / "appmanifest_1771300.acf" + before = manifest.read_text() + install_dir = library.common / "Game" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + def explode(self, *args, **kwargs): + raise OSError(28, "No space left on device") + + monkeypatch.setattr("pathlib.Path.write_text", explode) + with pytest.raises(OSError): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + # Original intact, and no temp file left behind. + assert manifest.read_text() == before + assert list(library.steamapps.glob("*.tmp*")) == [] + + +def test_check_ready_to_finish_blocks_when_no_game_content(tmp_path, monkeypatch): + """Cache filenames are not content: a dir holding only + .DepotDownloader/_.manifest must not be written as installed.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + assert any("no game content" in p for p in problems) + + +def test_check_ready_to_finish_blocks_unknown_public_build(tmp_path, monkeypatch): + """A None expected build means we cannot tell current from stale -- refuse + rather than silently skipping the comparison.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id=None, + ) + assert any("current public build" in p for p in problems) + + +def test_gate_blocks_before_staging_is_destroyed(tmp_path, monkeypatch): + """finish_manifest_manually raises when every cached depot is shared. The + CLI deletes Steam's staging tree first, so the gate must catch this while + refusing is still free -- otherwise a failed run destroys tens of GB of + resumable download state.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1001_555.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "999", "SharedDepots": {"1001": "228980"}}, + required_depots={"1001"}, + expected_build_id="999", + ) + assert any("nothing to record as installed" in p for p in problems) + + +def _fake_proc(tmp_path, entries): + """Build a /proc-shaped tree: {pid: exe_target_path_or_None}.""" + proc = tmp_path / "proc" + proc.mkdir() + for name, target in entries.items(): + entry = proc / name + entry.mkdir() + if target is not None: + real = tmp_path / "bin" / target + real.parent.mkdir(parents=True, exist_ok=True) + real.write_text("") + (entry / "exe").symlink_to(real) + return proc + + +def test_steam_client_pids_finds_native_and_flatpak(tmp_path, monkeypatch): + """Matching on the executable name only: an earlier version also demanded + 'ubuntu12_32' in the path, so a Flatpak Steam (/app/bin/steam) silently + failed open and the whole guard stopped guarding.""" + proc = _fake_proc( + tmp_path, + { + "111": "steam", + "222": "steamwebhelper", + "333": "steamcmd", + "444": "bash", + "self": "steam", + "555": None, + }, + ) + monkeypatch.setattr("steam_installer.recognize.PROC_DIR", proc) + + assert sorted(steam_client_pids()) == [111, 222] + + +def test_steam_client_pids_ignores_vanished_processes(tmp_path, monkeypatch): + """A pid can exit between listing /proc and resolving its exe.""" + proc = _fake_proc(tmp_path, {"111": "steam"}) + (proc / "999").mkdir() + (proc / "999" / "exe").symlink_to(tmp_path / "gone") + monkeypatch.setattr("steam_installer.recognize.PROC_DIR", proc) + + class _Boom(type(proc)): + pass + + assert steam_client_pids() == [111] + + +def test_run_steamcmd_validate_argv_and_result(monkeypatch): + seen = {} + + def fake_run_logged(cmd, logger, timeout=None): + seen["cmd"] = cmd + seen["timeout"] = timeout + return 0, "all good" + + monkeypatch.setattr("steam_installer.recognize.runlog.run_logged", fake_run_logged) + + result = run_steamcmd_validate(1771300, "krzys2000", logging.getLogger("t")) + + cmd = seen["cmd"] + # The platform flag must precede +login, and no force_install_dir may appear: + # it sets a LIBRARY ROOT and pointing it at the game dir caused the original + # "Invalid platform" failure. + assert cmd.index("+@sSteamCmdForcePlatformType") < cmd.index("+login") + assert "force_install_dir" not in " ".join(cmd) + assert cmd[cmd.index("+app_update") + 1 : cmd.index("+app_update") + 3] == [ + "1771300", + "validate", + ] + assert result.completed and result.returncode == 0 + assert result.output == "all good" + assert not result.crashed_on_install_script + + +def test_run_steamcmd_validate_detects_the_install_script_segfault(monkeypatch): + """The known steamcmd bug kills the process with SIGSEGV; checking the + signal beats grepping captured text for a message string.""" + monkeypatch.setattr( + "steam_installer.recognize.runlog.run_logged", + lambda cmd, logger, timeout=None: (-signal.SIGSEGV, ""), + ) + + result = run_steamcmd_validate(1, "u", logging.getLogger("t")) + + assert result.crashed_on_install_script + assert not result.completed + + +def test_ensure_installdir_correct_ignores_malformed_manifest(tmp_path, monkeypatch): + """Defensive guard: vdf.parse always yields a dict today, so inject a parser + that does not, to keep the guard honest if vdf ever changes.""" + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.write_text(vdf.dumps("AppState", {"installdir": "Stale"})) + before = manifest.read_text() + monkeypatch.setattr( + "steam_installer.recognize.vdf.parse", lambda text: {"AppState": "scalar"} + ) + + ensure_installdir_correct(library, 1771300, library.common / "Game") + + assert manifest.read_text() == before + + +def test_ensure_installdir_correct_is_a_noop_when_already_right(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.write_text(vdf.dumps("AppState", {"installdir": "Game"})) + + ensure_installdir_correct(library, 1771300, library.common / "Game") + + assert not manifest.with_suffix(".acf.bak").exists() + + +def test_manifest_reflects_real_install_false_on_malformed_appstate( + tmp_path, monkeypatch +): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", {"StateFlags": "4"}) + ) + monkeypatch.setattr( + "steam_installer.recognize.vdf.parse", lambda text: {"AppState": "scalar"} + ) + assert not manifest_reflects_real_install(library, 1771300, library.common / "Game") + + +def test_finish_manifest_rejects_malformed_appstate(tmp_path, monkeypatch): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1.manifest").write_bytes(b"x") + monkeypatch.setattr( + "steam_installer.recognize.vdf.parse", lambda text: {"AppState": "scalar"} + ) + + with pytest.raises(RecognizeError, match="not a valid AppState"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1) + + +def test_steam_client_pids_skips_unreadable_exe(tmp_path, monkeypatch): + """/proc//exe raises for processes we don't own; that must not abort + the scan and silently disable the guard.""" + proc = tmp_path / "proc" + (proc / "111").mkdir(parents=True) + (proc / "222").mkdir() + real = tmp_path / "bin" / "steam" + real.parent.mkdir(parents=True) + real.write_text("") + (proc / "222" / "exe").symlink_to(real) + monkeypatch.setattr("steam_installer.recognize.PROC_DIR", proc) + + original = Path.resolve + + def flaky(self, *args, **kwargs): + if self.parent.name == "111": + raise PermissionError("not owned by this user") + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", flaky) + + assert steam_client_pids() == [222] + + +def test_write_atomic_cleans_up_its_temp_file_on_failure(tmp_path, monkeypatch): + target = tmp_path / "appmanifest_1.acf" + target.write_text("original") + + def explode(self, *args, **kwargs): + raise OSError(28, "No space left on device") + + monkeypatch.setattr("pathlib.Path.write_text", explode) + with pytest.raises(OSError): + recognize._write_atomic(target, "new content") + + assert list(tmp_path.glob("*.tmp*")) == [] + + +def test_measure_content_size_skips_directories_and_symlinks(tmp_path): + """Directories have a size of their own, and a symlink would double-count + its target -- neither is game content Steam should see in SizeOnDisk.""" + install_dir = tmp_path / "Game" + (install_dir / "Data" / "Levels").mkdir(parents=True) + (install_dir / "Data" / "real.pak").write_bytes(b"x" * 250) + (install_dir / "Data" / "link.pak").symlink_to(install_dir / "Data" / "real.pak") + + assert measure_content_size(install_dir) == 250 diff --git a/_installer_import/tests/test_resolve.py b/_installer_import/tests/test_resolve.py new file mode 100644 index 0000000..e9a46a3 --- /dev/null +++ b/_installer_import/tests/test_resolve.py @@ -0,0 +1,99 @@ +import io +import json +import urllib.error + +import pytest + +from steam_installer import resolve + + +class _FakeResponse(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_search_parses_items(monkeypatch): + payload = {"items": [{"id": 1771300, "name": "Kingdom Come: Deliverance II"}]} + monkeypatch.setattr( + resolve.urllib.request, + "urlopen", + lambda url, timeout: _FakeResponse(json.dumps(payload).encode()), + ) + + results = resolve.search("kingdom come") + + assert results == [ + resolve.SearchResult(appid=1771300, name="Kingdom Come: Deliverance II") + ] + + +def test_search_quotes_the_term(monkeypatch): + seen = {} + + def fake_urlopen(url, timeout): + seen["url"] = url + return _FakeResponse(b'{"items": []}') + + monkeypatch.setattr(resolve.urllib.request, "urlopen", fake_urlopen) + + resolve.search("kingdom come/2") + + assert "kingdom%20come/2" in seen["url"] or "kingdom%20come%2F2" in seen["url"] + + +def test_search_missing_items_key_returns_empty(monkeypatch): + monkeypatch.setattr( + resolve.urllib.request, "urlopen", lambda url, timeout: _FakeResponse(b"{}") + ) + assert resolve.search("nothing") == [] + + +@pytest.mark.parametrize( + "exc", [urllib.error.URLError("tls blocked"), TimeoutError("timed out")] +) +def test_search_network_failure_becomes_search_unavailable(monkeypatch, exc): + """store.steampowered.com is known to reject scripted clients on some + networks -- that is not a code bug, so it gets its own exception type.""" + + def boom(url, timeout): + raise exc + + monkeypatch.setattr(resolve.urllib.request, "urlopen", boom) + + with pytest.raises(resolve.SearchUnavailable): + resolve.search("kingdom come") + + +def _results(): + return [ + resolve.SearchResult(appid=1, name="One"), + resolve.SearchResult(appid=2, name="Two"), + ] + + +def test_prompt_confirm_returns_the_chosen_result(monkeypatch): + monkeypatch.setattr("builtins.input", lambda *a: "2") + assert resolve.prompt_confirm(_results()).appid == 2 + + +def test_prompt_confirm_reprompts_until_valid(monkeypatch, capsys): + answers = iter(["", "banana", "9", "1"]) + monkeypatch.setattr("builtins.input", lambda *a: next(answers)) + + assert resolve.prompt_confirm(_results()).appid == 1 + assert capsys.readouterr().out.count("Invalid choice") == 3 + + +def test_prompt_confirm_zero_cancels(monkeypatch): + monkeypatch.setattr("builtins.input", lambda *a: "0") + with pytest.raises(resolve.ResolveError, match="Cancelled"): + resolve.prompt_confirm(_results()) + + +def test_prompt_confirm_rejects_empty_result_list(): + """Never guess an app: no matches must be an error, not a default pick.""" + with pytest.raises(resolve.ResolveError, match="No matches"): + resolve.prompt_confirm([]) diff --git a/_installer_import/tests/test_runlog.py b/_installer_import/tests/test_runlog.py new file mode 100644 index 0000000..61521db --- /dev/null +++ b/_installer_import/tests/test_runlog.py @@ -0,0 +1,210 @@ +import errno +import logging +import os +import subprocess +import sys + +import pytest + +from steam_installer import runlog + + +@pytest.fixture(autouse=True) +def isolated_log_dir(tmp_path, monkeypatch): + """Never write to the real ~/.local/state during tests.""" + monkeypatch.setattr(runlog, "LOG_DIR", tmp_path / "state") + yield + logger = logging.getLogger(runlog.LOGGER_NAME) + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + +def test_setup_creates_log_file_and_latest_symlink(): + log = runlog.setup(1771300) + + assert log.path.is_file() + assert "1771300" in log.path.name + latest = runlog.LOG_DIR / "latest.log" + assert latest.is_symlink() + assert latest.resolve() == log.path.resolve() + + +def test_setup_without_appid_still_names_the_file(): + log = runlog.setup(None) + assert "noappid" in log.path.name + + +def test_setup_twice_does_not_duplicate_handlers(): + """A second setup() in one process must not write every line twice.""" + runlog.setup(1) + log = runlog.setup(2) + log.logger.info("once") + assert log.path.read_text().count("once") == 1 + + +def test_run_logged_records_command_output_and_exit_code(capfd): + log = runlog.setup(1771300) + + returncode, output = runlog.run_logged( + ["bash", "-c", "echo to-stdout; echo to-stderr >&2; exit 7"], log.logger + ) + + assert returncode == 7 + assert "to-stdout" in output + assert "to-stderr" in output + + # Mirrored to the terminal... + assert "to-stdout" in capfd.readouterr().out + # ...and recorded in the log, along with the argv and exit code. + contents = log.path.read_text() + assert "to-stdout" in contents + assert "echo to-stdout" in contents + assert "exit=7" in contents + + +def test_run_logged_child_sees_a_tty(): + """The whole reason for the pty: steamcmd only prompts interactively for a + password when it believes it is attached to a terminal. A plain pipe makes + the prompt invisible and the run looks hung. + """ + log = runlog.setup(1771300) + _, output = runlog.run_logged( + [sys.executable, "-c", "import sys; print(sys.stdout.isatty())"], log.logger + ) + assert "True" in output + + +def test_run_logged_times_out(): + log = runlog.setup(1771300) + with pytest.raises(subprocess.TimeoutExpired): + runlog.run_logged(["sleep", "10"], log.logger, timeout=0.5) + + +def test_file_contents_records_manifest_verbatim(tmp_path): + log = runlog.setup(1771300) + manifest = tmp_path / "appmanifest_1771300.acf" + manifest.write_text('"AppState"\n{\n\t"StateFlags"\t\t"4"\n}\n') + + log.file_contents("Manifest before run", manifest) + + contents = log.path.read_text() + assert "Manifest before run" in contents + assert '"StateFlags"' in contents + + +def test_file_contents_survives_missing_file(tmp_path): + """A first-ever install has no manifest yet; that must not kill the run.""" + log = runlog.setup(1771300) + log.file_contents("Manifest before run", tmp_path / "absent.acf") + assert "unreadable" in log.path.read_text() + + +def test_run_logged_records_output_even_on_timeout(): + """A steamcmd hang at the 3600s timeout is exactly when the log matters + most; discarding what the child said before hanging defeats the point.""" + log = runlog.setup(1771300) + marker = "TIMEOUT" + "MARKER" + with pytest.raises(subprocess.TimeoutExpired) as excinfo: + runlog.run_logged( + ["bash", "-c", f'printf "%s\\n" "{marker}"; sleep 30'], + log.logger, + timeout=1.0, + ) + + contents = log.path.read_text() + assert "--- output" in contents + assert marker in contents + assert "TIMED OUT" in contents + assert marker.encode() in excinfo.value.output + + +def test_point_latest_at_survives_a_broken_log_dir(monkeypatch, tmp_path): + """A stale or unwritable latest.log must never fail an otherwise fine run.""" + log = runlog.setup(1) + latest = runlog.LOG_DIR / "latest.log" + latest.unlink() + latest.mkdir() # a directory where the symlink should go + + runlog._point_latest_at(log.path) # must not raise + + assert latest.is_dir() + + +def test_run_logged_closes_pty_fds_when_spawn_fails(monkeypatch): + """A leaked master/slave pair per failed spawn would exhaust the fd table.""" + log = runlog.setup(1) + before = len(os.listdir("/proc/self/fd")) + + with pytest.raises(FileNotFoundError): + runlog.run_logged(["definitely-not-a-real-binary-xyz"], log.logger) + + assert len(os.listdir("/proc/self/fd")) == before + + +def test_run_logged_stops_when_select_fails(monkeypatch): + """A closed master fd surfaces as OSError from select; end the relay.""" + log = runlog.setup(1) + monkeypatch.setattr( + runlog.select, "select", lambda *a, **kw: (_ for _ in ()).throw(OSError) + ) + + returncode, output = runlog.run_logged(["true"], log.logger) + + assert returncode == 0 + assert output == "" + + +def test_run_logged_reraises_unexpected_read_errors(monkeypatch): + """EIO means "child closed the pty" and ends the loop; anything else is a + real failure and must not be swallowed.""" + log = runlog.setup(1) + # Force select to always report readable so os.read is definitely reached. + monkeypatch.setattr(runlog.select, "select", lambda *a, **kw: ([a[0][0]], [], [])) + + def bad_read(fd, size): + raise OSError(errno.EBADF, "bad file descriptor") + + monkeypatch.setattr(runlog, "_read_pty", bad_read) + + with pytest.raises(OSError) as excinfo: + runlog.run_logged(["bash", "-c", "echo hi; sleep 0.2"], log.logger) + assert excinfo.value.errno == errno.EBADF + + +def test_run_logged_exits_when_child_dies_without_output(monkeypatch): + """select never reports readable, so the loop must notice via poll() that + the child is gone rather than spinning forever.""" + log = runlog.setup(1) + monkeypatch.setattr(runlog.select, "select", lambda *a, **kw: ([], [], [])) + + returncode, output = runlog.run_logged(["true"], log.logger) + + assert returncode == 0 + assert output == "" + + +def test_run_logged_handles_empty_read(monkeypatch): + """A zero-byte read is EOF on the pty.""" + log = runlog.setup(1) + real_read = runlog._read_pty + calls = {"n": 0} + + def once_then_eof(fd, size): + calls["n"] += 1 + if calls["n"] == 1: + return real_read(fd, size) + return b"" + + monkeypatch.setattr(runlog, "_read_pty", once_then_eof) + + returncode, _ = runlog.run_logged(["bash", "-c", "echo hi"], log.logger) + assert returncode == 0 + + +def test_run_logged_survives_a_child_that_writes_nothing(): + """Exercises the poll()-based exit when the pty never becomes readable.""" + log = runlog.setup(1) + returncode, output = runlog.run_logged(["true"], log.logger) + assert returncode == 0 + assert output == "" diff --git a/_installer_import/tests/test_vdf.py b/_installer_import/tests/test_vdf.py index e742f07..bab3a95 100644 --- a/_installer_import/tests/test_vdf.py +++ b/_installer_import/tests/test_vdf.py @@ -1,3 +1,5 @@ +import pytest + from steam_installer import vdf @@ -66,3 +68,80 @@ def test_roundtrip_dumps_then_parse(): def test_empty_input_returns_empty_dict(): assert vdf.parse("") == {} + + +def test_dumps_reescapes_backslashes_so_round_trip_is_stable(): + r"""Real regression: appmanifest_228980.acf (Steamworks Common + Redistributables) stores Windows paths like + `_CommonRedist\\vcredist\\2015\\installscript.vdf`. The tokenizer consumes + the escapes, so writing values back unescaped halved the backslashes on + every parse/dump cycle and silently corrupted the InstallScripts paths.""" + original = ( + '"AppState"\n{\n\t"InstallScripts"\n\t{\n' + '\t\t"228990"\t\t"_CommonRedist\\\\vcredist\\\\2015\\\\installscript.vdf"\n' + "\t}\n}\n" + ) + once = vdf.dumps("AppState", vdf.parse(original)["AppState"]) + assert once.strip() == original.strip() + + # And it must be a fixed point, not merely correct on the first pass. + twice = vdf.dumps("AppState", vdf.parse(once)["AppState"]) + assert twice == once + assert ( + vdf.parse(twice)["AppState"]["InstallScripts"]["228990"] + == "_CommonRedist\\vcredist\\2015\\installscript.vdf" + ) + + +def test_dumps_escapes_embedded_quotes(): + data = {"name": 'Game "Deluxe" Edition'} + assert vdf.parse(vdf.dumps("AppState", data))["AppState"] == data + + +def test_parse_error_message_includes_position(): + err = vdf.VdfParseError("boom", 7) + assert str(err) == "boom (at position 7)" + + +def test_parse_empty_input_returns_empty_dict(): + assert vdf.parse("") == {} + assert vdf.parse(" \n\t ") == {} + + +def test_parse_rejects_root_key_without_a_block(): + with pytest.raises(vdf.VdfParseError, match="expected '{' after root key"): + vdf.parse('"AppState"\n') + + +def test_parse_rejects_truncated_input_after_a_key(): + """A crash mid-write used to leave exactly this shape on disk.""" + with pytest.raises(vdf.VdfParseError, match="unexpected end of input"): + vdf.parse('"AppState"\n{\n\t"installdir"') + + +def test_parse_tolerates_unquoted_tokens(): + """Rare in Steam's own files, but the tokenizer accepts them.""" + assert vdf.parse("AppState\n{\n\tinstalldir\tGame\n}\n") == { + "AppState": {"installdir": "Game"} + } + + +def test_parse_ignores_comment_lines(): + text = '// leading comment\n"AppState"\n{\n\t"installdir"\t\t"Game"\n}\n' + assert vdf.parse(text) == {"AppState": {"installdir": "Game"}} + + +def test_parse_reads_scalar_value_after_key(): + """The non-block branch: a key whose value is a plain token.""" + assert vdf.parse('"AppState"\n{\n\t"StateFlags"\t\t"4"\n}\n') == { + "AppState": {"StateFlags": "4"} + } + + +def test_parse_tolerates_an_unterminated_block(): + """A truncated manifest (what a crash mid-write leaves behind) parses to + whatever was complete, rather than raising -- the caller's own validity + checks then reject it.""" + assert vdf.parse('"AppState"\n{\n\t"installdir"\t\t"Game"\n') == { + "AppState": {"installdir": "Game"} + } From 1b11de131b5120d450371e2340525f6615093940 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 14 Aug 2026 14:25:41 +0200 Subject: [PATCH 04/24] Add refactor_claude_todo.md: 250-line file-length cap A ready-to-use prompt describing the goal (every file at most 250 lines, code and prose alike), this repo's actual violations ranked by Fowler ROI (lines x commits/yr), how to split each kind, and the requirement to wire in the shared pre-commit gate so files stay small permanently. Generated from a measured survey of every repo; the gate itself lives in ~/utils (file_length/ + scripts/check_file_length.sh). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C7yZz45ezmzCB89Cv5gAtZ --- _installer_import/refactor_claude_todo.md | 83 +++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 _installer_import/refactor_claude_todo.md diff --git a/_installer_import/refactor_claude_todo.md b/_installer_import/refactor_claude_todo.md new file mode 100644 index 0000000..51ba1c9 --- /dev/null +++ b/_installer_import/refactor_claude_todo.md @@ -0,0 +1,83 @@ +# Refactor TODO — enforce the 250-line file cap + +> **This file is a ready-to-use prompt.** Paste it to Claude, or open this repo +> and say "do refactor_claude_todo". It is self-contained: everything needed to +> execute is below. Generated 2026-08-14 from a measured survey of every repo. + +## Goal + +Every file in this repo must be **at most 250 lines** — source, tests, and +prose (`.md`/`.txt`/`.rst`/`.tex`) alike — and must **stay** that way forever, +enforced by a gate that fails the commit, not by a note anyone can ignore. + +Why: a file that cannot be read in one piece forces re-reads and partial edits, +which is the single largest avoidable cost in an LLM-assisted workflow. Aim by +churn, not size alone — refactoring pays where code is read and changed often +(Fowler, *refactoring economic benefit*). + +## Scope in this repo + +- **4 files** currently exceed 250 lines (of 21 eligible files). +- **1,877 lines** sit in violation; longest file is **765 lines**. + +Exempt (do NOT split these): +- generated files — `*.g.dart`, `*.freezed.dart`, `*.gr.dart`, `**/l10n/generated/**`, + anything with a `GENERATED` header +- markup — `.html`, `.css`, `.scss` +- data files — `.json`, `.yaml`, `.csv`, wordlists and other data-ish `.txt` + (mean line length under 25 chars) + +## Violations, highest ROI first + +ROI = lines x commits in the last year. Work top-down; a long file nobody edits +has near-zero payoff and should not be first. + +| lines | commits/yr | kind | file | +|------:|-----------:|:-----|:-----| +| 765 | 3 | code | `tests/test_recognize.py` | +| 473 | 3 | code | `steam_installer/recognize.py` | +| 311 | 3 | code | `steam_installer/cli.py` | +| 328 | 1 | code | `tests/test_cli.py` | + +## How to split + +- **Python** — extract cohesive helpers into sibling modules; keep the public + API and imports stable. +- **Shell** — split into `lib/*.sh` sourced by a thin entry script. Keep + `set -euo pipefail` in each. +- **Dart / TypeScript** — extract widgets/components into their own files. +- **Tests** — split by test-group into sibling files + (`foo.test.ts` -> `foo.parsing.test.ts`, `foo.render.test.ts`). Coverage must + not drop. +- **Docs** — split into topic files under `docs/` with an index. For an + oversized `CLAUDE.md`, move detail into referenced docs so the + always-loaded part shrinks. + +**Do not** game the cap: no one-lining, no deleting tests, no moving code into +an exempt extension, no `# noqa`-style suppressions. + +## Make it permanent (required — this is the point) + +A refactor without a gate silently regrows. Before this task is done: + +1. Wire the shared gate `~/utils/scripts/check_file_length.sh` into this repo's + `.pre-commit-config.yaml` as a local hook. If the repo has no pre-commit + config, add a minimal one. +2. The hook checks **files in the commit** (not the whole tree), so unrelated + commits never break, and it **fails** — exit 1, not a warning. +3. No baseline file and no allowlist. Those are suppressions. +4. If this repo has CI (`.github/workflows`), add the same check there so it + also fails on push. + +## Done condition + +- `bash ~/utils/scripts/check_file_length.sh --all` from this repo root exits 0. +- The repo's own test suite and coverage bar are still green. +- `pre-commit run --files ` passes. +- A deliberately over-250-line test file, staged, makes `git commit` **fail**. +- For a deployed daemon/app: the entry point still actually runs. + +## Verify + +Run the repo's own suite, then run the program itself and confirm the +behaviour is unchanged. Testing is the last step, never the first. From 0815708e02bda92a9de95fc86df47da760699e39 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Mon, 24 Aug 2026 15:53:43 +0200 Subject: [PATCH 05/24] Adopt the four markdown namespaces and gate them Renames this repo's markdown onto README.md / CLAUDE*.md / DOCS*.md / TODO*.md so a future session can tell a live task from a finished record without reading the file to find out. TODO files carry the literal "REMOVE ME AFTER FINISH" and are deleted when the work lands. Enforced by the md-naming pre-commit hook and CI workflow, both delegating to the shared gate in kuhyx/utils so this repo's idea of a valid name cannot drift from every other repo's. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01A47aX5DJwmA3QFn3pkEWi7 --- _installer_import/.pre-commit-config.yaml | 17 ++++++++++ ...claude_todo.md => TODO-file-length-250.md} | 0 _installer_import/scripts/check_md_naming.sh | 32 +++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 _installer_import/.pre-commit-config.yaml rename _installer_import/{refactor_claude_todo.md => TODO-file-length-250.md} (100%) create mode 100755 _installer_import/scripts/check_md_naming.sh diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml new file mode 100644 index 0000000..4c6b62e --- /dev/null +++ b/_installer_import/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +# Pre-commit hooks for this repo. +# +# Created when the shared markdown-naming gate was rolled out. The checks +# are `language: system` because they are plain bash already on this +# machine; pre-commit must not build an isolated env for them. +repos: + - repo: local + hooks: + # Four markdown namespaces (README / CLAUDE* / DOCS* / TODO*), plus the + # rule that a TODO carries "REMOVE ME AFTER FINISH" and nothing else may. + # Shared gate in ~/utils owns the namespaces and the exemption list, so + # one repo's idea of a valid name cannot drift from every other repo's. + - id: md-naming + name: markdown naming convention + entry: scripts/check_md_naming.sh + language: system + files: \.(md|markdown)$ diff --git a/_installer_import/refactor_claude_todo.md b/_installer_import/TODO-file-length-250.md similarity index 100% rename from _installer_import/refactor_claude_todo.md rename to _installer_import/TODO-file-length-250.md diff --git a/_installer_import/scripts/check_md_naming.sh b/_installer_import/scripts/check_md_naming.sh new file mode 100755 index 0000000..50deaa0 --- /dev/null +++ b/_installer_import/scripts/check_md_naming.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============================================================================ +# Fail if any markdown file breaks the shared naming convention. +# +# Thin delegate to the shared gate in ~/utils, which owns the four namespaces +# (README / CLAUDE* / DOCS* / TODO*), the "REMOVE ME AFTER FINISH" marker rule +# and the exemption list. Copying that logic here is what lets one repo's idea +# of a valid name drift from every other repo's -- so this script only locates +# the shared checker and forwards its arguments. +# +# Usage: +# scripts/check_md_naming.sh [ ...] # pre-commit passes these +# scripts/check_md_naming.sh --all # whole tree, from cwd +# ============================================================================ + +set -euo pipefail + +readonly SHARED_GATE="${UTILS_ROOT:-$HOME/utils}/scripts/check_md_naming.sh" + +main() { + if [[ ! -x "$SHARED_GATE" ]]; then + echo "Error: shared markdown-naming gate not found at $SHARED_GATE" >&2 + echo " Clone github.com/kuhyx/utils to ~/utils, or set" >&2 + echo " UTILS_ROOT to where it lives." >&2 + exit 1 + fi + + exec bash "$SHARED_GATE" "$@" +} + +main "$@" From a4813b09bce28d4d53c8fe4f65dc3b93e5a77d2d Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Mon, 24 Aug 2026 15:56:24 +0200 Subject: [PATCH 06/24] Add the removal marker to every TODO file The rename commit staged the git mv but not the accompanying content edit, so these TODO files landed without the "REMOVE ME AFTER FINISH" line the md-naming gate requires. Repos whose hook was already live refused that commit; repos without it accepted a file the gate would now reject. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01A47aX5DJwmA3QFn3pkEWi7 --- _installer_import/TODO-file-length-250.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/_installer_import/TODO-file-length-250.md b/_installer_import/TODO-file-length-250.md index 51ba1c9..95a5f86 100644 --- a/_installer_import/TODO-file-length-250.md +++ b/_installer_import/TODO-file-length-250.md @@ -13,7 +13,7 @@ enforced by a gate that fails the commit, not by a note anyone can ignore. Why: a file that cannot be read in one piece forces re-reads and partial edits, which is the single largest avoidable cost in an LLM-assisted workflow. Aim by churn, not size alone — refactoring pays where code is read and changed often -(Fowler, *refactoring economic benefit*). +(Fowler, _refactoring economic benefit_). ## Scope in this repo @@ -21,6 +21,7 @@ churn, not size alone — refactoring pays where code is read and changed often - **1,877 lines** sit in violation; longest file is **765 lines**. Exempt (do NOT split these): + - generated files — `*.g.dart`, `*.freezed.dart`, `*.gr.dart`, `**/l10n/generated/**`, anything with a `GENERATED` header - markup — `.html`, `.css`, `.scss` @@ -32,12 +33,12 @@ Exempt (do NOT split these): ROI = lines x commits in the last year. Work top-down; a long file nobody edits has near-zero payoff and should not be first. -| lines | commits/yr | kind | file | -|------:|-----------:|:-----|:-----| -| 765 | 3 | code | `tests/test_recognize.py` | -| 473 | 3 | code | `steam_installer/recognize.py` | -| 311 | 3 | code | `steam_installer/cli.py` | -| 328 | 1 | code | `tests/test_cli.py` | +| lines | commits/yr | kind | file | +| ----: | ---------: | :--- | :----------------------------- | +| 765 | 3 | code | `tests/test_recognize.py` | +| 473 | 3 | code | `steam_installer/recognize.py` | +| 311 | 3 | code | `steam_installer/cli.py` | +| 328 | 1 | code | `tests/test_cli.py` | ## How to split @@ -81,3 +82,5 @@ A refactor without a gate silently regrows. Before this task is done: Run the repo's own suite, then run the program itself and confirm the behaviour is unchanged. Testing is the last step, never the first. + +REMOVE ME AFTER FINISH From a93043c694cd7ab9c63a9d3a0f6a4cf9deb2d56d Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 28 Aug 2026 16:34:14 +0200 Subject: [PATCH 07/24] Add the dependency-freshness gate Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/dependency-freshness.yml | 62 +++++++++++++++++++ _installer_import/.pre-commit-config.yaml | 22 +++++++ .../scripts/check_dependency_freshness.sh | 34 ++++++++++ 3 files changed, 118 insertions(+) create mode 100644 _installer_import/.github/workflows/dependency-freshness.yml create mode 100755 _installer_import/scripts/check_dependency_freshness.sh diff --git a/_installer_import/.github/workflows/dependency-freshness.yml b/_installer_import/.github/workflows/dependency-freshness.yml new file mode 100644 index 0000000..23a0a73 --- /dev/null +++ b/_installer_import/.github/workflows/dependency-freshness.yml @@ -0,0 +1,62 @@ +name: dependency freshness + +# Every dependency on its ecosystem's newest STABLE release, enforced on push +# as well as at commit time. The local pre-commit hook only sees the manifests +# in a commit and can be skipped entirely by a fresh clone that never ran +# `pre-commit install`; this runs `--all` over the whole repo. +# +# `--strict` is CI-only. Offline the gate degrades to its cached answer and +# passes, because `--no-verify` is banned locally and a hook that hard-failed +# on a flaky connection would leave no way to commit. A runner is always +# online, so here an undeterminable answer is a real failure. +# +# Expect this to go red for reasons this repo's commit did not cause: an +# upstream release overnight is enough. That is the chosen behaviour -- the +# alternative is a gate that only notices staleness when someone remembers to +# look. + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + dependency-freshness: + name: dependencies on newest stable + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # The gate lives in the shared utils repo so every repo resolves the + # same "latest stable" the same way. Checking it out beats vendoring a + # copy, which is exactly how a gate and its survey start disagreeing. + - name: Check out the shared gate + uses: actions/checkout@v4 + with: + repository: kuhyx/utils + path: .utils + + # "3.x", never a pinned minor: the Python target IS the interpreter the + # gate runs under (registries/toolchain.python_installed), so a runner + # held a release behind this machine reports a repo's `requires-python` + # as excluding "latest" while the same check passes locally. flax-docs-mcp + # hit exactly that -- `>=3.14` green here, red on a 3.13 runner. + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install the gate's two runtime dependencies + run: python -m pip install --upgrade pyyaml packaging + + - name: Cache registry lookups + uses: actions/cache@v4 + with: + path: ~/.cache/dep-freshness + key: dep-freshness-${{ hashFiles('**/pubspec.yaml', '**/pyproject.toml', + '**/requirements*.txt', '**/package.json', '**/Cargo.toml', '**/go.mod') }} + restore-keys: dep-freshness- + + - name: Every dependency is on newest stable + run: bash .utils/scripts/check_dependency_freshness.sh --all --strict diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml index 4c6b62e..9c1a169 100644 --- a/_installer_import/.pre-commit-config.yaml +++ b/_installer_import/.pre-commit-config.yaml @@ -15,3 +15,25 @@ repos: entry: scripts/check_md_naming.sh language: system files: \.(md|markdown)$ + + # Every dependency on its ecosystem's newest STABLE release, exact-pinned + # so the question is mechanically answerable at all. + # + # `files:` is what keeps this cheap: a commit touching only source makes + # ZERO network requests, and manifest commits mostly hit a 6h disk cache. + # + # Runs WITHOUT --strict on purpose. Offline it degrades to the cached + # answer and passes with a loud banner; `--no-verify` is banned here, so + # a hook that hard-failed on a flaky connection would leave no way to + # commit at all. CI, always online, is the strict authority. + - id: dependency-freshness + name: dependencies on newest stable + entry: scripts/check_dependency_freshness.sh + language: system + pass_filenames: true + files: >- + (pubspec\.(yaml|lock)|pyproject\.toml|requirements.*\.txt|uv\.lock + |package\.json|package-lock\.json|pnpm-lock\.yaml + |Cargo\.(toml|lock)|go\.(mod|sum) + |\.fvmrc|\.nvmrc|\.python-version + |\.github/workflows/.*\.ya?ml)$ diff --git a/_installer_import/scripts/check_dependency_freshness.sh b/_installer_import/scripts/check_dependency_freshness.sh new file mode 100755 index 0000000..5cb0d78 --- /dev/null +++ b/_installer_import/scripts/check_dependency_freshness.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============================================================================ +# Fail if any dependency in the commit is behind its ecosystem's newest stable. +# +# Thin delegate to the shared gate in ~/utils, which owns the registry +# adapters, the pre-release rules and the allowlist semantics. Copying that +# logic here is what lets one repo's idea of "current" drift from every other +# repo's -- so this script only locates the shared checker and forwards its +# arguments. +# +# Usage: +# scripts/check_dependency_freshness.sh [ ...] # pre-commit +# scripts/check_dependency_freshness.sh --all --strict # CI +# +# Exit: 0 current | 1 behind | 2 allowlist wrong | 3 undeterminable (offline) +# ============================================================================ + +set -euo pipefail + +readonly SHARED_GATE="${UTILS_ROOT:-$HOME/utils}/scripts/check_dependency_freshness.sh" + +main() { + if [[ ! -x "$SHARED_GATE" ]]; then + echo "Error: shared dependency-freshness gate not found at $SHARED_GATE" >&2 + echo " Clone github.com/kuhyx/utils to ~/utils, or set" >&2 + echo " UTILS_ROOT to where it lives." >&2 + exit 1 + fi + + exec bash "$SHARED_GATE" "$@" +} + +main "$@" From 0f2609fc52cd63b31b6f4c6ca039bc403c247b10 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 28 Aug 2026 19:38:25 +0200 Subject: [PATCH 08/24] One Python, always the newest; cover the credited-steamcmd branch Co-Authored-By: Claude Opus 5 (1M context) --- _installer_import/pyproject.toml | 2 +- _installer_import/tests/test_cli.py | 41 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/_installer_import/pyproject.toml b/_installer_import/pyproject.toml index 43b0307..a6609d3 100644 --- a/_installer_import/pyproject.toml +++ b/_installer_import/pyproject.toml @@ -2,7 +2,7 @@ name = "steam-game-installer" version = "0.1.0" description = "Fast Steam game installs: depotdownloader for speed, steamcmd for recognition." -requires-python = ">=3.11" +requires-python = ">=3.14" dependencies = [] [project.scripts] diff --git a/_installer_import/tests/test_cli.py b/_installer_import/tests/test_cli.py index 0170056..c460dfd 100644 --- a/_installer_import/tests/test_cli.py +++ b/_installer_import/tests/test_cli.py @@ -177,6 +177,47 @@ def test_clean_steamcmd_run_short_circuits(steam, monkeypatch): assert called == [] +def test_a_complete_install_after_a_failed_steamcmd_is_not_credited_to_it( + steam, monkeypatch, capsys +): + """A valid manifest does not mean steamcmd is why it is valid. + + steamcmd can fail to even log in while the install was already complete on + arrival. Reporting that as "finished cleanly" credits a step that did + nothing and hides a real authentication failure, so the non-zero exit has + to say so out loud. + """ + library, _ = steam + _stub_download(monkeypatch) + + def fake_validate(appid, username, logger): + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps( + "AppState", + { + "StateFlags": "4", + "installdir": "KingdomComeDeliverance2", + "InstalledDepots": {"1771303": {"manifest": "1"}}, + "BytesToDownload": "1", + "BytesDownloaded": "1", + }, + ) + ) + return recognize.ValidateResult(True, False, 5) + + monkeypatch.setattr(cli.recognize, "run_steamcmd_validate", fake_validate) + called = [] + monkeypatch.setattr( + cli.recognize, "finish_manifest_manually", lambda **kw: called.append(1) + ) + + assert cli.main(ARGV) == 0 + out = capsys.readouterr().out + assert "steamcmd exited 5" in out + assert "finished cleanly" not in out + assert called == [] + + def test_user_declining_confirmation_aborts(steam, monkeypatch): monkeypatch.setattr("builtins.input", lambda *a: "n") called = [] From 4c35dc9b63770a65dc4fae6d9123d9ab358673fd Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 11 Sep 2026 18:46:33 +0200 Subject: [PATCH 09/24] Re-point paths for the ~ reorganisation (home-tidy) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01AwtowrowDXx6ccmWVPkCaF --- _installer_import/.pre-commit-config.yaml | 2 +- _installer_import/TODO-file-length-250.md | 4 ++-- _installer_import/install.sh | 2 +- _installer_import/scripts/check_dependency_freshness.sh | 6 +++--- _installer_import/scripts/check_md_naming.sh | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml index 9c1a169..e76039b 100644 --- a/_installer_import/.pre-commit-config.yaml +++ b/_installer_import/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: hooks: # Four markdown namespaces (README / CLAUDE* / DOCS* / TODO*), plus the # rule that a TODO carries "REMOVE ME AFTER FINISH" and nothing else may. - # Shared gate in ~/utils owns the namespaces and the exemption list, so + # Shared gate in ~/src/utils owns the namespaces and the exemption list, so # one repo's idea of a valid name cannot drift from every other repo's. - id: md-naming name: markdown naming convention diff --git a/_installer_import/TODO-file-length-250.md b/_installer_import/TODO-file-length-250.md index 95a5f86..9f33c98 100644 --- a/_installer_import/TODO-file-length-250.md +++ b/_installer_import/TODO-file-length-250.md @@ -61,7 +61,7 @@ an exempt extension, no `# noqa`-style suppressions. A refactor without a gate silently regrows. Before this task is done: -1. Wire the shared gate `~/utils/scripts/check_file_length.sh` into this repo's +1. Wire the shared gate `~/src/utils/scripts/check_file_length.sh` into this repo's `.pre-commit-config.yaml` as a local hook. If the repo has no pre-commit config, add a minimal one. 2. The hook checks **files in the commit** (not the whole tree), so unrelated @@ -72,7 +72,7 @@ A refactor without a gate silently regrows. Before this task is done: ## Done condition -- `bash ~/utils/scripts/check_file_length.sh --all` from this repo root exits 0. +- `bash ~/src/utils/scripts/check_file_length.sh --all` from this repo root exits 0. - The repo's own test suite and coverage bar are still green. - `pre-commit run --files ` passes. - A deliberately over-250-line test file, staged, makes `git commit` **fail**. diff --git a/_installer_import/install.sh b/_installer_import/install.sh index e41cbe9..491fc39 100755 --- a/_installer_import/install.sh +++ b/_installer_import/install.sh @@ -11,7 +11,7 @@ readonly SCRIPT_DIR missing=0 if ! command -v depotdownloader >/dev/null 2>&1; then - echo "missing: depotdownloader (see ~/depotdownloader)" >&2 + echo "missing: depotdownloader (see ~/vendor/depotdownloader)" >&2 missing=1 fi diff --git a/_installer_import/scripts/check_dependency_freshness.sh b/_installer_import/scripts/check_dependency_freshness.sh index 5cb0d78..7715d3c 100755 --- a/_installer_import/scripts/check_dependency_freshness.sh +++ b/_installer_import/scripts/check_dependency_freshness.sh @@ -3,7 +3,7 @@ # ============================================================================ # Fail if any dependency in the commit is behind its ecosystem's newest stable. # -# Thin delegate to the shared gate in ~/utils, which owns the registry +# Thin delegate to the shared gate in ~/src/utils, which owns the registry # adapters, the pre-release rules and the allowlist semantics. Copying that # logic here is what lets one repo's idea of "current" drift from every other # repo's -- so this script only locates the shared checker and forwards its @@ -18,12 +18,12 @@ set -euo pipefail -readonly SHARED_GATE="${UTILS_ROOT:-$HOME/utils}/scripts/check_dependency_freshness.sh" +readonly SHARED_GATE="${UTILS_ROOT:-$HOME/src/utils}/scripts/check_dependency_freshness.sh" main() { if [[ ! -x "$SHARED_GATE" ]]; then echo "Error: shared dependency-freshness gate not found at $SHARED_GATE" >&2 - echo " Clone github.com/kuhyx/utils to ~/utils, or set" >&2 + echo " Clone github.com/kuhyx/utils to ~/src/utils, or set" >&2 echo " UTILS_ROOT to where it lives." >&2 exit 1 fi diff --git a/_installer_import/scripts/check_md_naming.sh b/_installer_import/scripts/check_md_naming.sh index 50deaa0..7847c56 100755 --- a/_installer_import/scripts/check_md_naming.sh +++ b/_installer_import/scripts/check_md_naming.sh @@ -3,7 +3,7 @@ # ============================================================================ # Fail if any markdown file breaks the shared naming convention. # -# Thin delegate to the shared gate in ~/utils, which owns the four namespaces +# Thin delegate to the shared gate in ~/src/utils, which owns the four namespaces # (README / CLAUDE* / DOCS* / TODO*), the "REMOVE ME AFTER FINISH" marker rule # and the exemption list. Copying that logic here is what lets one repo's idea # of a valid name drift from every other repo's -- so this script only locates @@ -16,12 +16,12 @@ set -euo pipefail -readonly SHARED_GATE="${UTILS_ROOT:-$HOME/utils}/scripts/check_md_naming.sh" +readonly SHARED_GATE="${UTILS_ROOT:-$HOME/src/utils}/scripts/check_md_naming.sh" main() { if [[ ! -x "$SHARED_GATE" ]]; then echo "Error: shared markdown-naming gate not found at $SHARED_GATE" >&2 - echo " Clone github.com/kuhyx/utils to ~/utils, or set" >&2 + echo " Clone github.com/kuhyx/utils to ~/src/utils, or set" >&2 echo " UTILS_ROOT to where it lives." >&2 exit 1 fi From 15f647f6398ed9ce781cdb03af48f3957a2072e7 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 11 Sep 2026 21:38:16 +0200 Subject: [PATCH 10/24] Install any game from an appid, a URL or a name; adopt Steam's partial downloads `./run.sh ` is now the entry point. It installs its own dependencies (steamcmd via yay, DepotDownloader from the SteamRE release, a local .venv) and works out what the argument is: an appid, a Steam URL, an exact title, or part of a title. Every route is confirmed through steamcmd's own app_info before anything acts on it. Name search moves to steamcommunity.com/actions/SearchApps, which works, with the storefront API kept as a fallback -- it TLS-rejects scripted clients here, so search had been effectively dead. DNS on this box fails transiently about one lookup in three, so each backend is retried before giving up. An exact title now wins over Steam's own ranking, which puts "Portal 2" above "Portal"; an ambiguous name shows a picker, or exits 2 rather than guessing. Two problems found while building this, both now handled: Anonymous steamcmd has no licence for a paid app and does not say so. It transfers nothing, prints "Success! App '284160' fully installed.", exits 0, and writes StateFlags=4 with SizeOnDisk=0 and no depots. A running Steam client then reads that, decides the app is installed but stale, and downloads the whole game by itself. `ghost.py` detects that state -- validated read-only against all ten manifests in a real library, where the nine genuine installs are untouched and only the fabricated one matches -- and `--repair-ghosts` clears it away. Repair uses rmdir, never rmtree, so a non-empty directory survives a wrong diagnosis. And Steam's own in-progress downloads are no longer wasted. `takeover.py` hardlinks `steamapps/downloading//` into the install directory before depotdownloader runs; depotdownloader hash-checks what is already on disk (ContentDownloader.cs skips a file when neededChunks is empty) and fetches only the remainder. Nothing is deleted up front -- the staging tree is cleared only after the install is verified -- so a failed run still leaves Steam able to resume. The trees share inodes, so an in-place chunk patch corrects both. cli.py drops from 311 to 244 lines, back under the repo's 250-line cap, by moving the install flow to flow.py and the manifest finish to finalize.py. install.sh had a latent bug: `pip install -e .[dev]` only warns about a missing extra and exits 0, so the `|| fallback` that installed pytest and coverage never ran. A fresh venv had no test tools at all; the existing one only worked because they had been installed by hand earlier. Now named outright and asserted. 246 tests, 100% statement and branch coverage. Verified live: all four input forms resolve to 284160, an exact "Portal" resolves to 400 and not 620, an ambiguous name exits 2, ghost detection flags nothing on nine real installs, and a from-scratch venv bootstraps and runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Sgkb3wH6MU9fYquX328ovT --- _installer_import/README.md | 71 ++- _installer_import/install.sh | 171 ++++++-- _installer_import/run.sh | 102 +++++ _installer_import/steam_installer/cli.py | 406 ++++++++---------- _installer_import/steam_installer/config.py | 63 +++ _installer_import/steam_installer/deps.py | 31 ++ _installer_import/steam_installer/finalize.py | 92 ++++ _installer_import/steam_installer/flow.py | 182 ++++++++ _installer_import/steam_installer/ghost.py | 188 ++++++++ _installer_import/steam_installer/resolve.py | 123 +++++- _installer_import/steam_installer/steamctl.py | 72 ++++ _installer_import/steam_installer/takeover.py | 118 +++++ _installer_import/steam_installer/target.py | 112 +++++ _installer_import/tests/conftest.py | 114 ++++- _installer_import/tests/test_cli.py | 95 ++-- _installer_import/tests/test_cli_modes.py | 51 +++ _installer_import/tests/test_cli_steam.py | 205 +++++++++ _installer_import/tests/test_config.py | 95 ++++ _installer_import/tests/test_deps.py | 31 ++ _installer_import/tests/test_ghost.py | 159 +++++++ _installer_import/tests/test_ghost_repair.py | 74 ++++ _installer_import/tests/test_resolve.py | 161 ++++++- _installer_import/tests/test_steamctl.py | 58 +++ _installer_import/tests/test_takeover.py | 134 ++++++ _installer_import/tests/test_target.py | 178 ++++++++ 25 files changed, 2722 insertions(+), 364 deletions(-) create mode 100755 _installer_import/run.sh create mode 100644 _installer_import/steam_installer/config.py create mode 100644 _installer_import/steam_installer/deps.py create mode 100644 _installer_import/steam_installer/finalize.py create mode 100644 _installer_import/steam_installer/flow.py create mode 100644 _installer_import/steam_installer/ghost.py create mode 100644 _installer_import/steam_installer/steamctl.py create mode 100644 _installer_import/steam_installer/takeover.py create mode 100644 _installer_import/steam_installer/target.py create mode 100644 _installer_import/tests/test_cli_modes.py create mode 100644 _installer_import/tests/test_cli_steam.py create mode 100644 _installer_import/tests/test_config.py create mode 100644 _installer_import/tests/test_deps.py create mode 100644 _installer_import/tests/test_ghost.py create mode 100644 _installer_import/tests/test_ghost_repair.py create mode 100644 _installer_import/tests/test_steamctl.py create mode 100644 _installer_import/tests/test_takeover.py create mode 100644 _installer_import/tests/test_target.py diff --git a/_installer_import/README.md b/_installer_import/README.md index 550e0ee..08c1823 100644 --- a/_installer_import/README.md +++ b/_installer_import/README.md @@ -5,14 +5,73 @@ better than the Steam client), then make the Steam client accept the result as a real install so **Play** works. ```bash -./install.sh # checks depotdownloader + steamcmd are present -pip install -e . -steam-game-installer --appid 1771300 --username +./run.sh BeamNG.drive ``` -**Close the Steam client first.** Steam owns `appmanifest_.acf` while it +That is the whole thing. `run.sh` installs its own dependencies (steamcmd via +`yay`, DepotDownloader from the SteamRE release, and a local `.venv`) on first +use, then works out what you meant. All four of these install the same game: + +```bash +./run.sh 284160 # an AppID +./run.sh https://steamcommunity.com/app/284160/guides/ # any Steam URL +./run.sh BeamNG.drive # the exact name +./run.sh BeamNG # part of the name +``` + +An exact title always wins, so `./run.sh Portal` installs Portal and not +Portal 2 — even though Steam's own search ranks the sequel first. An ambiguous +name shows a numbered picker, and in a non-interactive run it lists the +candidates and exits 2 rather than guessing. + +Useful flags: `--dry-run` (resolve and report, change nothing), `--yes` +(no prompts), `--repair-ghosts` (see below), `--username` (remembered after the +first run, in `~/.config/steam-game-installer/config`; never a password). + +**Steam gets closed for you.** Steam owns `appmanifest_.acf` while it runs and rewrites it from its own in-memory state, so anything written -underneath a live client is discarded. The tool refuses to start if it sees one. +underneath a live client is discarded. The tool asks, shuts it down with +`steam -shutdown`, and restarts it afterwards. It never sends a signal: a client +that refuses to exit aborts the run instead. + +## It takes over a download Steam already started + +Steam stages an in-progress download into `steamapps/downloading//`, laid +out exactly like the finished game. Those bytes are yours — you waited for them +— so the tool **hardlinks them into place** before calling depotdownloader, +which hash-checks what is already on disk and fetches only the remainder. + +Hardlinks, not moves and not copies. Nothing is deleted up front: the staging +tree is cleared only at the very end, so a failed run still leaves Steam able to +resume its own download. Linking is free, where copying a 47 GB tree would cost +47 GB. + +Because the two trees share inodes, depotdownloader patching a partial file in +place also rewrites Steam's copy. That is deliberate — both names converge on +the same corrected content — but it does mean the staged bytes are working data, +not a frozen backup. On ext4 there is no reflink that would give you both. + +## Ghost installs + +`steamcmd +login anonymous +app_update ` has no licence for the app. +It does not say so. It transfers nothing, prints `Success! App '...' fully +installed.`, exits 0, and writes a manifest claiming `StateFlags=4` with +`SizeOnDisk=0` and an empty `InstalledDepots`. + +That manifest is not harmless: a running Steam client reads it, concludes the +app is installed but out of date, and downloads the entire game by itself. + +```bash +./run.sh --repair-ghosts +``` + +finds them across every library and clears them away — but only the *cold* ones. +A ghost with staged content behind it is a real partial download; it is reported +and left alone, to be adopted by a normal install instead. Detection requires +all of: the manifest claims installed, `InstalledDepots` is empty, `SizeOnDisk` +is `0`, there are zero content bytes on disk, and there is no `.DepotDownloader` +state. Repair uses `rmdir`, never `rmtree`, so a non-empty directory cannot be +deleted even if the diagnosis were wrong. ## Run logs @@ -50,7 +109,7 @@ The tool refuses to mark an app installed, and exits non-zero, unless: optional and shared depots — KCD2 lists 12 depots but an english/windows install only ever fetches 3); - the manifest carries a real `buildid` matching the current public build; -- no Steam client is running; +- no Steam client is running (it is closed first, and restarted afterwards); - the manifest reads back as a real install *after* being written. This matters: an earlier version wrote `StateFlags=4` and reported success after diff --git a/_installer_import/install.sh b/_installer_import/install.sh index 491fc39..2ae395e 100755 --- a/_installer_import/install.sh +++ b/_installer_import/install.sh @@ -1,50 +1,169 @@ #!/bin/bash -# Checks for depotdownloader and steamcmd on PATH. steamcmd is AUR-only and -# needs an interactive license/sudo prompt, so it's never installed for you -- -# this just tells you the command to run. + +# ============================================================================ +# Install everything this tool shells out to, then set up the dev environment. +# +# steamcmd AUR package, via yay +# depotdownloader official self-contained release from SteamRE/DepotDownloader +# .venv local virtualenv with the package installed editable +# +# Idempotent: every step no-ops when its target is already in place, so run.sh +# can call it on every run. `yay` shells out to `sudo pacman`, so installing +# steamcmd may prompt for a password -- that is the one interactive step. +# ============================================================================ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR +readonly VENV="${SCRIPT_DIR}/.venv" +readonly STAMP="${VENV}/.deps-ok" +readonly DD_REPO="SteamRE/DepotDownloader" +readonly DD_HOME="${XDG_DATA_HOME:-$HOME/.local/share}/steam-game-installer/depotdownloader" +readonly BIN_DIR="${HOME}/.local/bin" -missing=0 +CHECK_ONLY=0 +TEMP_DIR="" -if ! command -v depotdownloader >/dev/null 2>&1; then - echo "missing: depotdownloader (see ~/vendor/depotdownloader)" >&2 - missing=1 -fi +cleanup() { + if [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]]; then + rm -rf -- "$TEMP_DIR" + fi +} +trap cleanup EXIT -if ! command -v steamcmd >/dev/null 2>&1; then - echo "missing: steamcmd -- run: yay -S steamcmd" >&2 - missing=1 -fi +usage() { + echo "Usage: install.sh [OPTIONS]" + echo "Options:" + echo " --check-only Report missing dependencies without installing anything" + echo " -h, --help Show this help" + exit 0 +} +die() { + echo "error: $*" >&2 + exit 1 +} + +ensure_steamcmd() { + if command -v steamcmd >/dev/null 2>&1; then + return 0 + fi + if [[ "$CHECK_ONLY" -eq 1 ]]; then + echo "missing: steamcmd -- run: yay -S steamcmd" >&2 + return 1 + fi + command -v yay >/dev/null 2>&1 || + die "steamcmd is missing and yay is not installed. Install it, then: yay -S steamcmd" + echo "Installing steamcmd (yay may ask for your sudo password)..." + yay -S --needed --noconfirm --answerdiff None --answeredit None steamcmd + command -v steamcmd >/dev/null 2>&1 || die "steamcmd still not on PATH after install" +} + +# Build from source only if the published binary can't be fetched. The upstream +# global.json pins the SDK to 9.0.x with rollForward=latestMinor, which will NOT +# roll forward to dotnet 10 -- so this needs dotnet-sdk-9.0 specifically. +build_depotdownloader() { + local src="${HOME}/vendor/depotdownloader" + [[ -d "$src" ]] || die "could not download DepotDownloader and no source tree at $src" + command -v yay >/dev/null 2>&1 || die "need yay to install dotnet-sdk-9.0" + pacman -Qq dotnet-sdk-9.0 >/dev/null 2>&1 || + yay -S --needed --noconfirm dotnet-sdk-9.0 + (cd "$src" && dotnet build -c Release) + ln -sfn "${src}/DepotDownloader/bin/Release/net9.0/DepotDownloader" \ + "${BIN_DIR}/depotdownloader" +} + +ensure_depotdownloader() { + if command -v depotdownloader >/dev/null 2>&1; then + return 0 + fi + if [[ "$CHECK_ONLY" -eq 1 ]]; then + echo "missing: depotdownloader -- run ./install.sh" >&2 + return 1 + fi + + mkdir -p "$BIN_DIR" + echo "Fetching the latest DepotDownloader release..." + local url="" + url="$(curl -fsSL "https://api.github.com/repos/${DD_REPO}/releases/latest" | + grep -oE '"browser_download_url": *"[^"]*linux-x64\.zip"' | + cut -d'"' -f4 || true)" + + if [[ -n "$url" ]]; then + TEMP_DIR="$(mktemp -d)" + mkdir -p "$DD_HOME" + # bsdtar ships with libarchive, a pacman dependency; unzip may not exist. + curl -fsSL -o "${TEMP_DIR}/dd.zip" "$url" && + bsdtar -xf "${TEMP_DIR}/dd.zip" -C "$DD_HOME" && + chmod +x "${DD_HOME}/DepotDownloader" && + ln -sfn "${DD_HOME}/DepotDownloader" "${BIN_DIR}/depotdownloader" + else + echo "Release download unavailable; falling back to a source build." >&2 + build_depotdownloader + fi + + command -v depotdownloader >/dev/null 2>&1 || + die "depotdownloader installed to ${BIN_DIR} but is not on PATH. Add ${BIN_DIR} to PATH." +} + +# The suite requires 100% coverage (pyproject: fail_under = 100), and one test +# runs `python -m steam_installer.cli` in a subprocess. Coverage in a child is +# only recorded if it calls coverage.process_startup() at interpreter start, +# which is what this .pth does. Without it a clean clone reports 99% and fails. setup_dev_env() { - # The test suite requires 100% coverage (pyproject: fail_under = 100), and - # one test runs `python -m steam_installer.cli` in a subprocess. Coverage in - # a child process is only recorded if it calls coverage.process_startup() at - # interpreter start, which is what this .pth does. Without it a clean clone - # reports 99% and the suite fails. - local venv="${SCRIPT_DIR}/.venv" - [[ -d "$venv" ]] || return 0 + if [[ ! -x "${VENV}/bin/python" ]]; then + echo "Creating ${VENV}..." + python3 -m venv "$VENV" + fi - local python="${venv}/bin/python" - "$python" -m pip install -q -e "${SCRIPT_DIR}[dev]" 2>/dev/null || - "$python" -m pip install -q -e "$SCRIPT_DIR" pytest pytest-cov + local python="${VENV}/bin/python" + "$python" -m pip install -q --upgrade pip + # Named outright rather than via an extra: pyproject declares none, and pip + # only *warns* about a missing extra before exiting 0 -- so the old + # `[dev] || fallback` form silently installed no test tools at all on a + # fresh venv, and the .pth below then failed to import coverage. + "$python" -m pip install -q -e "$SCRIPT_DIR" pytest pytest-cov coverage local site_packages site_packages="$("$python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" cat >"${site_packages}/steam_installer_coverage.pth" <<-'PTH' import os, coverage; os.environ.get("COVERAGE_PROCESS_START") and coverage.process_startup() PTH + "$python" -c 'import pytest, coverage' || + die "dev environment incomplete: pytest/coverage did not install" echo "Dev environment ready (subprocess coverage enabled)." } -if [[ "$missing" -eq 0 ]]; then +main() { + local missing=0 + ensure_steamcmd || missing=1 + ensure_depotdownloader || missing=1 + + if [[ "$CHECK_ONLY" -eq 1 ]]; then + [[ "$missing" -eq 0 ]] && echo "All dependencies present." + exit "$missing" + fi + + setup_dev_env echo "All dependencies present." -fi + : >"$STAMP" +} -setup_dev_env +while [[ $# -gt 0 ]]; do + case $1 in + --check-only) + CHECK_ONLY=1 + shift + ;; + -h | --help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done -exit "$missing" +main diff --git a/_installer_import/run.sh b/_installer_import/run.sh new file mode 100755 index 0000000..27ff431 --- /dev/null +++ b/_installer_import/run.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# ============================================================================ +# Install a Steam game from whatever you happen to have: an AppID, a Steam URL, +# or the game's name. +# +# ./run.sh 284160 +# ./run.sh https://steamcommunity.com/app/284160/guides/ +# ./run.sh BeamNG.drive +# ./run.sh BeamNG +# +# Dependencies install themselves on first run. Every other argument is passed +# straight through to the CLI, so `./run.sh --help` lists the full set. +# ============================================================================ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +readonly VENV_PY="${SCRIPT_DIR}/.venv/bin/python" +readonly STAMP="${SCRIPT_DIR}/.venv/.deps-ok" +readonly INSTALLER="${SCRIPT_DIR}/install.sh" + +SKIP_DEPS="${SGI_SKIP_DEPS:-0}" +DEPS_ONLY=0 +ARGS=() + +usage() { + echo "Usage: run.sh [TARGET] [OPTIONS]" + echo + echo "TARGET is any of:" + echo " 284160 an AppID" + echo " https://steamcommunity.com/app/284160/... a Steam URL" + echo " BeamNG.drive the exact name" + echo " BeamNG part of the name" + echo + echo "run.sh options:" + echo " --no-deps Skip the dependency check (also: SGI_SKIP_DEPS=1)" + echo " --deps-only Install dependencies and exit" + echo " -h, --help Show this help, then the installer's own options" + echo + if [[ -x "$VENV_PY" ]]; then + "$VENV_PY" -m steam_installer.cli --help + fi + exit 0 +} + +# Dependencies are only re-checked when the stamp is missing or older than the +# files that decide what gets installed, so a normal run adds no startup cost. +deps_are_current() { + [[ -x "$VENV_PY" ]] || return 1 + [[ -f "$STAMP" ]] || return 1 + [[ "$STAMP" -nt "$INSTALLER" ]] || return 1 + [[ "$STAMP" -nt "${SCRIPT_DIR}/pyproject.toml" ]] || return 1 + "$VENV_PY" -c 'import steam_installer' >/dev/null 2>&1 +} + +main() { + if [[ "$SKIP_DEPS" != "1" ]] && ! deps_are_current; then + "$INSTALLER" + fi + + if [[ "$DEPS_ONLY" -eq 1 ]]; then + exit 0 + fi + + [[ -x "$VENV_PY" ]] || { + echo "error: no virtualenv at ${VENV_PY}. Run ./install.sh" >&2 + exit 1 + } + + # `-m` rather than the console script, so a stale .venv shim can never + # shadow the working tree. + exec "$VENV_PY" -m steam_installer.cli "${ARGS[@]}" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --no-deps) + SKIP_DEPS=1 + shift + ;; + --deps-only) + DEPS_ONLY=1 + shift + ;; + -h | --help) + usage + ;; + --) + shift + ARGS+=("$@") + break + ;; + *) + ARGS+=("$1") + shift + ;; + esac +done + +main diff --git a/_installer_import/steam_installer/cli.py b/_installer_import/steam_installer/cli.py index 7cbb849..d44b707 100644 --- a/_installer_import/steam_installer/cli.py +++ b/_installer_import/steam_installer/cli.py @@ -1,66 +1,71 @@ -"""Entry point: resolve -> confirm -> fast download -> make Steam recognize it.""" +"""Entry point: resolve -> confirm -> adopt what exists -> fast download -> recognize.""" from __future__ import annotations import argparse -import shutil import sys -import time -from pathlib import Path -from steam_installer import appinfo, depotdl, recognize, resolve, runlog -from steam_installer.library import ( - SteamLibrary, - find_steam_root, - list_libraries, - pick_install_target, +from steam_installer import ( + config, + depotdl, + deps, + flow, + ghost, + resolve, + runlog, + takeover, + target, ) - - -def _require_tool(name: str, install_hint: str) -> None: - if shutil.which(name) is None: - print(f"error: `{name}` not found on PATH. {install_hint}", file=sys.stderr) - sys.exit(1) - - -def _resolve_appid(args: argparse.Namespace) -> int: - """Prefer an explicit --appid; fall back to search, which is best-effort - (store.steampowered.com is known to reject scripted clients on some - networks -- see resolve.py). Either way, the final confirmation uses the - real name from steamcmd's own app info, not whatever the search returned. - """ - if args.appid is not None: - return args.appid - - try: - results = resolve.search(args.name) - except resolve.SearchUnavailable as exc: - print(f"Store search unavailable ({exc}).", file=sys.stderr) - print( - "Find the numeric AppID yourself (e.g. from the store URL) and pass --appid.", - file=sys.stderr, - ) - sys.exit(1) - - chosen = resolve.prompt_confirm(results) - return chosen.appid +from steam_installer.library import find_steam_root, list_libraries, pick_install_target def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Download an owned Steam game fast, then make Steam recognize it." ) - parser.add_argument("name", nargs="?", default=None, help="Game name to search for") parser.add_argument( - "--appid", type=int, default=None, help="Steam AppID (skips search)" + "target", + nargs="?", + default=None, + metavar="TARGET", + help=( + "AppID (284160), a Steam URL " + "(https://steamcommunity.com/app/284160/guides/), or a game name " + "(BeamNG.drive, or just BeamNG)" + ), + ) + parser.add_argument( + "--appid", type=int, default=None, help="Steam AppID (skips detection)" + ) + parser.add_argument( + "--username", + default=None, + help="Steam account username (default: $STEAM_USERNAME, then the saved one)", ) - parser.add_argument("--username", required=True, help="Steam account username") parser.add_argument( "--language", default=None, help="Language depot to fetch (default: prompts)" ) parser.add_argument( "--max-downloads", type=int, default=depotdl.DEFAULT_MAX_DOWNLOADS ) + parser.add_argument( + "--yes", + action="store_true", + help="Assume yes for every prompt. Ambiguous names still fail rather than guess.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Resolve the target and report what would happen. Changes nothing.", + ) + parser.add_argument( + "--repair-ghosts", + action="store_true", + help=( + "Scan every library for the fake 'installed' manifests anonymous " + "steamcmd leaves behind, and clear them away." + ), + ) parser.add_argument( "--skip-steamcmd", action="store_true", @@ -84,10 +89,11 @@ def main(argv: list[str] | None = None) -> int: parser = _build_parser() args = parser.parse_args(argv) - if args.appid is None and args.name is None: - parser.error("provide a game name to search for, or --appid directly") + if not args.repair_ghosts and args.appid is None and args.target is None: + parser.error("provide an AppID, a Steam URL, a game name, or --repair-ghosts") - log = runlog.setup(args.appid, verbose=args.verbose) + literal = target.parse_literal(args.target) if args.target else None + log = runlog.setup(args.appid or (literal[0] if literal else None), args.verbose) logger = log.logger # Printed up front as well as at the end: if the run dies badly, the path # still has to be on screen to copy-paste. @@ -102,198 +108,136 @@ def main(argv: list[str] | None = None) -> int: return 1 -def _run(args: argparse.Namespace, log: runlog.RunLog) -> int: +def _is_interactive(args: argparse.Namespace) -> bool: + return sys.stdin.isatty() and not args.yes + + +def _ask(args: argparse.Namespace, interactive: bool, question: str) -> bool: + if args.yes: + return True + if not interactive: + return False + return input(question).strip().lower() == "y" + + +def _repair_ghosts(log: runlog.RunLog) -> int: + """Report every ghost manifest; delete only the ones with nothing behind them.""" logger = log.logger + reports = ghost.scan(list_libraries(find_steam_root())) + if not reports: + logger.info("No ghost installs found.") + return 0 - _require_tool("depotdownloader", "See ~/depotdownloader.") - if not args.skip_steamcmd: - _require_tool("steamcmd", "Run: yay -S steamcmd") - - appid = _resolve_appid(args) - metadata = appinfo.get_metadata(appid) - logger.info("Confirmed: %s (appid %s)", metadata.name, metadata.appid) - answer = input("Proceed? [y/N]: ").strip().lower() - if answer != "y": - logger.info("Cancelled.") - return 1 - - steam_root = find_steam_root() - libraries = list_libraries(steam_root) - target = pick_install_target(libraries, metadata.installdir) - logger.info("Install target: %s", target) - - language = args.language - if language is None: - language = input("Language [english]: ").strip() or "english" - - # Refuse early rather than after an hour of downloading: a live Steam client - # rewrites the manifest from its own state and discards whatever we write. - running = recognize.steam_client_pids() - if running: - logger.error( - "The Steam client is running (pid %s). It owns appmanifest_%s.acf and " - "will overwrite anything this tool writes. Close Steam, then re-run.", - ", ".join(str(p) for p in running), - metadata.appid, - ) - logger.info("Run log (send this): %s", log.path) - return 1 - - depot_plan = appinfo.get_depot_plan(metadata.appid, language) - expected_build_id = appinfo.get_build_id(metadata.appid) - log.section("Plan") - logger.debug("language=%s max_downloads=%s", language, args.max_downloads) - logger.debug("required depots: %s", sorted(depot_plan.required)) - logger.debug("shared depots: %s", sorted(depot_plan.shared)) - logger.debug("expected buildid: %s", expected_build_id) - - library = next(lib for lib in libraries if lib.common == target.parent) - manifest_path = library.steamapps / f"appmanifest_{metadata.appid}.acf" - log.file_contents("Manifest before run", manifest_path) - - logger.info("Downloading via depotdownloader (fast path)...") - log.section("depotdownloader") - returncode = depotdl.download( - appid=metadata.appid, - install_dir=target, - username=args.username, - language=language, - max_downloads=args.max_downloads, - logger=logger, - ) - if returncode != 0: - logger.error( - "depotdownloader failed (exit %d) -- not touching the manifest.", returncode - ) - logger.info("Run log (send this): %s", log.path) - return 1 - - log.section("depotdownloader cache") - for depot_id, mid in sorted( - recognize.discover_cached_depot_manifests(target).items() - ): - logger.debug("depot %s -> manifest %s", depot_id, mid) - - recognize.ensure_installdir_correct(library, metadata.appid, target) - - if not args.skip_steamcmd: - logger.info("Reconciling with steamcmd (validate)...") - log.section("steamcmd app_update validate") - result = recognize.run_steamcmd_validate(metadata.appid, args.username, logger) - log.file_contents("Manifest after steamcmd", manifest_path) - - # Never trust steamcmd's exit code alone -- verify the manifest actually - # reflects a real install before believing it. A "clean" exit has - # produced a manifest steamcmd itself hadn't finished updating before. - if ( - not result.crashed_on_install_script - and recognize.manifest_reflects_real_install( - library, metadata.appid, target + for report in reports: + logger.info("Ghost: appid %s (%s)", report.appid, report.manifest_path.name) + for reason in report.reasons: + logger.info(" - %s", reason) + if report.is_warm: + logger.warning( + " Steam has %.2f GB staged for this app. Those bytes are a real " + "partial download, so nothing is deleted -- install it normally " + "(this tool adopts them) instead of repairing it.", + report.staging_bytes / 1e9, ) - ): - logger.info("steamcmd finished cleanly -- Steam should show Play.") - logger.info("Run log: %s", log.path) - return 0 - - if result.crashed_on_install_script: - logger.info( - "steamcmd hit the known install-script crash; finishing the manifest manually..." - ) - else: - logger.info( - "steamcmd exited %d but the manifest doesn't reflect a real install; " - "checking whether it is safe to finish it manually...", - result.returncode, - ) - - return _finish( - log, - library, - metadata, - target, - depot_plan, - expected_build_id, - skipped_steamcmd=args.skip_steamcmd, - ) - - -def _finish( - log: runlog.RunLog, - library: SteamLibrary, - metadata: appinfo.AppMetadata, - target: Path, - depot_plan: appinfo.DepotPlan, - expected_build_id: str | None, - skipped_steamcmd: bool, -) -> int: - """Hand-finish the manifest, but only if every precondition really holds.""" - logger = log.logger - manifest_path = library.steamapps / f"appmanifest_{metadata.appid}.acf" - - if not manifest_path.is_file(): - # Reachable two ways, so don't blame steamcmd unconditionally: under - # --skip-steamcmd it never runs at all, and nothing else in this tool - # creates a manifest from scratch. - logger.error( - "No %s exists, so there is nothing to finish. This tool only ever " - "patches a manifest Steam or steamcmd already wrote; it never " - "fabricates one. %s", - manifest_path, - ( - "Re-run without --skip-steamcmd so steamcmd creates it (it writes " - "one even when it crashes)." - if skipped_steamcmd - else "steamcmd writes one even when it crashes, so its absence " - "means steamcmd never ran far enough." - ), - ) - logger.info("Run log (send this): %s", log.path) - return 1 - - app_state = recognize.read_app_state(manifest_path) - problems = recognize.check_ready_to_finish( - install_dir=target, - app_state=app_state, - required_depots=depot_plan.required, - expected_build_id=expected_build_id, - ) - if problems: - # Fail closed. Writing StateFlags=4 here is what previously turned a - # steamcmd run that did nothing at all into a reported success. - logger.error("Refusing to mark %s as installed:", metadata.name) - for problem in problems: - logger.error(" - %s", problem) - logger.info("Run log (send this): %s", log.path) - return 1 - - # Do this before writing the manifest: a StateFlags=4 manifest alongside a - # staging tree full of pending chunks is the same contradiction that made - # Steam resume the download last time. - freed = recognize.clear_steam_staging(library, metadata.appid) - if freed: - logger.info("Cleared %.1f GB of stale Steam staging data.", freed / 1e9) - - written = recognize.finish_manifest_manually( - library=library, - appid=metadata.appid, - install_dir=target, - now_unix=int(time.time()), - ) - log.file_contents("Manifest after manual finish", written) - - # Re-read from disk: claiming success without checking the write landed is - # the same fail-open mistake in a different place. - if not recognize.manifest_reflects_real_install(library, metadata.appid, target): - logger.error( - "Wrote %s but it still does not read back as a real install.", written - ) - logger.info("Run log (send this): %s", log.path) - return 1 - - logger.info("Wrote %s -- launch Steam and confirm Play works.", written) + continue + ghost.repair(report, logger) logger.info("Run log: %s", log.path) return 0 +def _describe(logger, metadata, install_target, report, staged) -> int: + """--dry-run output: everything the real run would act on, and nothing done.""" + logger.info("Install target: %s", install_target) + if report.is_ghost: + logger.warning("This app currently has a GHOST manifest:") + for reason in report.reasons: + logger.warning(" - %s", reason) + if staged is not None: + logger.info( + "Steam has %.2f GB staged in %s (%d files) -- these would be adopted, " + "not re-downloaded.", + staged.bytes_staged / 1e9, + staged.staging_dir, + staged.file_count, + ) + logger.info("Dry run: nothing was changed.") + return 0 + + +def _run(args: argparse.Namespace, log: runlog.RunLog) -> int: + logger = log.logger + interactive = _is_interactive(args) + + if args.repair_ghosts: + return _repair_ghosts(log) + + try: + deps.require(skip_steamcmd=args.skip_steamcmd) + resolved = target.resolve_target( + args.target, args.appid, interactive=interactive + ) + except deps.DependencyError as exc: + logger.error("%s", exc) + return 1 + except (target.TargetError, resolve.ResolveError) as exc: + logger.error("%s", exc) + return 2 + + metadata = resolved.metadata + logger.info( + "Confirmed: %s (appid %s, matched by %s)", + metadata.name, + metadata.appid, + resolved.source, + ) + + libraries = list_libraries(find_steam_root()) + install_target = pick_install_target(libraries, metadata.installdir) + library = next(lib for lib in libraries if lib.common == install_target.parent) + report = ghost.inspect(library, metadata.appid) + staged = takeover.find_staged(library, metadata.appid, install_target) + + if args.dry_run: + return _describe(logger, metadata, install_target, report, staged) + + if report.is_ghost and not report.is_warm: + logger.error( + "appid %s has a ghost manifest (a fake 'installed' state with nothing " + "behind it). Clear it first: ./run.sh --repair-ghosts", + metadata.appid, + ) + return 1 + + try: + username = config.resolve_username(args.username, interactive=interactive) + except config.ConfigError as exc: + logger.error("%s", exc) + return 1 + + logger.info("Install target: %s", install_target) + if not _ask(args, interactive, "Proceed? [y/N]: "): + logger.info("Cancelled (use --yes for a non-interactive run).") + return 1 + + language = args.language + if language is None: + language = ( + (input("Language [english]: ").strip() or "english") + if interactive + else "english" + ) + + return flow.install( + args=args, + log=log, + library=library, + metadata=metadata, + install_target=install_target, + staged=staged, + username=username, + language=language, + ) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/_installer_import/steam_installer/config.py b/_installer_import/steam_installer/config.py new file mode 100644 index 0000000..a86be4d --- /dev/null +++ b/_installer_import/steam_installer/config.py @@ -0,0 +1,63 @@ +"""Remember the Steam username so every run doesn't ask for it again. + +Only the username is ever persisted. Passwords and Steam Guard codes stay +where they belong: with steamcmd's own cached login token (prompted through +the pty in `runlog.run_logged`) and depotdownloader's `-remember-password`. +Writing either of those to a dotfile would be a downgrade in security over +what Steam's own tooling already does. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +CONFIG_DIR = Path.home() / ".config" / "steam-game-installer" +CONFIG_PATH = CONFIG_DIR / "config" +USERNAME_KEY = "username" +ENV_VAR = "STEAM_USERNAME" + + +class ConfigError(Exception): + pass + + +def load_username() -> str | None: + """The saved username, or None if nothing has been saved yet.""" + if not CONFIG_PATH.is_file(): + return None + for line in CONFIG_PATH.read_text().splitlines(): + key, sep, value = line.partition("=") + if sep and key.strip() == USERNAME_KEY: + return value.strip() or None + return None + + +def save_username(username: str) -> None: + """Persist the username at 0600, replacing any previous value.""" + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + CONFIG_PATH.write_text(f"{USERNAME_KEY}={username}\n") + # Not a secret, but it names an account; no reason for it to be world-readable. + CONFIG_PATH.chmod(0o600) + + +def resolve_username(cli_value: str | None, *, interactive: bool) -> str: + """--username > $STEAM_USERNAME > saved config > prompt (and save the answer).""" + if cli_value: + return cli_value + env_value = os.environ.get(ENV_VAR, "").strip() + if env_value: + return env_value + saved = load_username() + if saved: + return saved + if not interactive: + raise ConfigError( + "No Steam username available. Pass --username, set " + f"${ENV_VAR}, or run once interactively to save one." + ) + entered = input("Steam username: ").strip() + if not entered: + raise ConfigError("No Steam username entered.") + save_username(entered) + return entered diff --git a/_installer_import/steam_installer/deps.py b/_installer_import/steam_installer/deps.py new file mode 100644 index 0000000..38e8f82 --- /dev/null +++ b/_installer_import/steam_installer/deps.py @@ -0,0 +1,31 @@ +"""External binaries this tool shells out to, and a clear error when they're missing.""" + +from __future__ import annotations + +import shutil + +TOOL_HINTS = { + "depotdownloader": "Run ./install.sh, which fetches it from the SteamRE release.", + "steamcmd": "Run ./install.sh, or: yay -S steamcmd", +} + + +class DependencyError(Exception): + pass + + +def missing_tools(*, skip_steamcmd: bool) -> list[str]: + """Which required binaries are not on PATH, in the order they'd be used.""" + needed = ["depotdownloader"] + if not skip_steamcmd: + needed.append("steamcmd") + return [tool for tool in needed if shutil.which(tool) is None] + + +def require(*, skip_steamcmd: bool) -> None: + missing = missing_tools(skip_steamcmd=skip_steamcmd) + if missing: + details = "; ".join( + f"`{tool}` not found -- {TOOL_HINTS[tool]}" for tool in missing + ) + raise DependencyError(details) diff --git a/_installer_import/steam_installer/finalize.py b/_installer_import/steam_installer/finalize.py new file mode 100644 index 0000000..dd95413 --- /dev/null +++ b/_installer_import/steam_installer/finalize.py @@ -0,0 +1,92 @@ +"""Hand-finish the appmanifest, but only when every precondition really holds. + +Split out of cli.py so the entry point stays under the repo's 250-line cap. +The logic is unchanged: it patches a manifest steamcmd or Steam already +wrote, and refuses rather than fabricating one. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +from steam_installer import appinfo, recognize, runlog +from steam_installer.library import SteamLibrary + + +def finish( + log: runlog.RunLog, + library: SteamLibrary, + metadata: appinfo.AppMetadata, + target: Path, + depot_plan: appinfo.DepotPlan, + expected_build_id: str | None, + skipped_steamcmd: bool, +) -> int: + logger = log.logger + manifest_path = library.steamapps / f"appmanifest_{metadata.appid}.acf" + + if not manifest_path.is_file(): + # Reachable two ways, so don't blame steamcmd unconditionally: under + # --skip-steamcmd it never runs at all, and nothing else in this tool + # creates a manifest from scratch. + logger.error( + "No %s exists, so there is nothing to finish. This tool only ever " + "patches a manifest Steam or steamcmd already wrote; it never " + "fabricates one. %s", + manifest_path, + ( + "Re-run without --skip-steamcmd so steamcmd creates it (it writes " + "one even when it crashes)." + if skipped_steamcmd + else "steamcmd writes one even when it crashes, so its absence " + "means steamcmd never ran far enough." + ), + ) + logger.info("Run log (send this): %s", log.path) + return 1 + + app_state = recognize.read_app_state(manifest_path) + problems = recognize.check_ready_to_finish( + install_dir=target, + app_state=app_state, + required_depots=depot_plan.required, + expected_build_id=expected_build_id, + ) + if problems: + # Fail closed. Writing StateFlags=4 here is what previously turned a + # steamcmd run that did nothing at all into a reported success. + logger.error("Refusing to mark %s as installed:", metadata.name) + for problem in problems: + logger.error(" - %s", problem) + logger.info("Run log (send this): %s", log.path) + return 1 + + # Do this before writing the manifest: a StateFlags=4 manifest alongside a + # staging tree full of pending chunks is the same contradiction that made + # Steam resume the download last time. By this point depotdownloader has + # already filled common/, so the staged copies are genuinely redundant. + freed = recognize.clear_steam_staging(library, metadata.appid) + if freed: + logger.info("Cleared %.1f GB of now-redundant Steam staging data.", freed / 1e9) + + written = recognize.finish_manifest_manually( + library=library, + appid=metadata.appid, + install_dir=target, + now_unix=int(time.time()), + ) + log.file_contents("Manifest after manual finish", written) + + # Re-read from disk: claiming success without checking the write landed is + # the same fail-open mistake in a different place. + if not recognize.manifest_reflects_real_install(library, metadata.appid, target): + logger.error( + "Wrote %s but it still does not read back as a real install.", written + ) + logger.info("Run log (send this): %s", log.path) + return 1 + + logger.info("Wrote %s -- launch Steam and confirm Play works.", written) + logger.info("Run log: %s", log.path) + return 0 diff --git a/_installer_import/steam_installer/flow.py b/_installer_import/steam_installer/flow.py new file mode 100644 index 0000000..2fad7e1 --- /dev/null +++ b/_installer_import/steam_installer/flow.py @@ -0,0 +1,182 @@ +"""Orchestrate one install: close Steam, adopt staged bytes, download, recognize. + +Split out of cli.py so the entry point stays under the repo's 250-line cap. +The manifest-finishing half lives in `finalize.py`. +""" + +from __future__ import annotations + +from pathlib import Path + +from steam_installer import ( + appinfo, + depotdl, + recognize, + runlog, + steamctl, + takeover, +) +from steam_installer.finalize import finish +from steam_installer.library import SteamLibrary + + +def _prepare_steam(args, logger) -> bool | None: + """Close Steam if it's running. Returns whether it was, or None to abort. + + Steam owns the manifest and is actively writing the staging tree, so it has + to be out of the way. Closing someone's Steam is not something to do + silently, so it is confirmed unless --yes was passed. + + The caller's own confirmation gate means we only get here with --yes or on + a tty, so prompting is always safe at this point. + """ + if not recognize.steam_client_pids(): + return False + if not args.yes: + answer = input("Steam is running and must be closed. Close it now? [y/N]: ") + if answer.strip().lower() != "y": + logger.info("Cancelled.") + return None + try: + return steamctl.stop(logger) + except steamctl.SteamStillRunning as exc: + logger.error("%s", exc) + return None + + +def install( + *, + args, + log: runlog.RunLog, + library: SteamLibrary, + metadata: appinfo.AppMetadata, + install_target: Path, + staged: takeover.Staged | None, + username: str, + language: str, +) -> int: + """Adopt anything Steam already fetched, download the rest, make it count.""" + logger = log.logger + + steam_was_running = _prepare_steam(args, logger) + if steam_was_running is None: + return 1 + try: + return _download_and_recognize( + args, log, library, metadata, install_target, staged, username, language + ) + finally: + if steam_was_running: + steamctl.start(logger) + + +def _download_and_recognize( + args, + log: runlog.RunLog, + library: SteamLibrary, + metadata: appinfo.AppMetadata, + install_target: Path, + staged: takeover.Staged | None, + username: str, + language: str, +) -> int: + logger = log.logger + + if staged is not None: + log.section("Adopting Steam's staged download") + logger.info( + "Steam already staged %.2f GB for this app (%d files). Hardlinking it " + "into place so depotdownloader verifies instead of refetching.", + staged.bytes_staged / 1e9, + staged.file_count, + ) + install_target.mkdir(parents=True, exist_ok=True) + takeover.adopt(staged, logger) + + depot_plan = appinfo.get_depot_plan(metadata.appid, language) + expected_build_id = appinfo.get_build_id(metadata.appid) + log.section("Plan") + logger.debug("language=%s max_downloads=%s", language, args.max_downloads) + logger.debug("required depots: %s", sorted(depot_plan.required)) + logger.debug("shared depots: %s", sorted(depot_plan.shared)) + logger.debug("expected buildid: %s", expected_build_id) + + manifest_path = library.steamapps / f"appmanifest_{metadata.appid}.acf" + log.file_contents("Manifest before run", manifest_path) + + logger.info("Downloading via depotdownloader (fast path)...") + log.section("depotdownloader") + returncode = depotdl.download( + appid=metadata.appid, + install_dir=install_target, + username=username, + language=language, + max_downloads=args.max_downloads, + logger=logger, + ) + if returncode != 0: + logger.error( + "depotdownloader failed (exit %d) -- not touching the manifest.", returncode + ) + logger.info("Run log (send this): %s", log.path) + return 1 + + log.section("depotdownloader cache") + for depot_id, mid in sorted( + recognize.discover_cached_depot_manifests(install_target).items() + ): + logger.debug("depot %s -> manifest %s", depot_id, mid) + + recognize.ensure_installdir_correct(library, metadata.appid, install_target) + + if not args.skip_steamcmd: + logger.info("Reconciling with steamcmd (validate)...") + log.section("steamcmd app_update validate") + result = recognize.run_steamcmd_validate(metadata.appid, username, logger) + log.file_contents("Manifest after steamcmd", manifest_path) + + # Never trust steamcmd's exit code alone -- verify the manifest actually + # reflects a real install before believing it. A "clean" exit has + # produced a manifest steamcmd itself hadn't finished updating before. + if ( + not result.crashed_on_install_script + and recognize.manifest_reflects_real_install( + library, metadata.appid, install_target + ) + ): + # The manifest being valid does not mean steamcmd is why. It can + # already have been correct on arrival while steamcmd failed to even + # log in -- reporting that as "steamcmd finished cleanly" credits a + # step that did nothing and hides a real authentication failure. + if result.returncode == 0: + logger.info("steamcmd finished cleanly -- Steam should show Play.") + else: + logger.warning( + "steamcmd exited %d, but %s is already a complete install " + "-- leaving it as is. Steam should show Play.", + result.returncode, + manifest_path.name, + ) + logger.info("Run log: %s", log.path) + return 0 + + if result.crashed_on_install_script: + logger.info( + "steamcmd hit the known install-script crash; finishing the manifest manually..." + ) + else: + logger.info( + "steamcmd exited %d but the manifest doesn't reflect a real install; " + "checking whether it is safe to finish it manually...", + result.returncode, + ) + + return finish( + log, + library, + metadata, + install_target, + depot_plan, + expected_build_id, + skipped_steamcmd=args.skip_steamcmd, + ) diff --git a/_installer_import/steam_installer/ghost.py b/_installer_import/steam_installer/ghost.py new file mode 100644 index 0000000..c15ce83 --- /dev/null +++ b/_installer_import/steam_installer/ghost.py @@ -0,0 +1,188 @@ +"""Detect the fake "fully installed" state anonymous steamcmd leaves behind. + +`steamcmd +login anonymous +app_update ` has no licence for the +app. It does not say so. It transfers nothing, prints + + Update state (0x0) unknown, progress: 0.00 (0 / 0) + Success! App '284160' fully installed. + +exits 0, and writes a manifest claiming StateFlags=4 with SizeOnDisk=0 and an +empty InstalledDepots block. Reproduced on this machine on 2026-09-11. + +That manifest is not harmless. A running Steam client reads it, concludes the +app is installed but out of date, and starts downloading the whole game by +itself -- which is exactly what happened, to the tune of 22 GB. + +Two outcomes, and only one of them deletes anything: + +* **Cold ghost** -- the lie and nothing else. Safe to clear away. +* **Warm ghost** -- the same manifest, but Steam has since staged real content + into `steamapps/downloading//`. Those bytes are worth keeping: the + whole point of this tool is to make installs faster, and deleting a partial + download to start over is the opposite. Warm ghosts are routed to + `takeover`, never to `repair`. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass +from pathlib import Path + +from steam_installer import recognize, vdf +from steam_installer.library import SteamLibrary + +INSTALLED_STATE_FLAG = 4 + + +@dataclass(frozen=True) +class GhostReport: + appid: int + manifest_path: Path + install_dir: Path | None + content_bytes: int + staging_bytes: int + reasons: tuple[str, ...] + + @property + def is_ghost(self) -> bool: + return bool(self.reasons) + + @property + def is_warm(self) -> bool: + """Steam has staged real content for this app; it must not be deleted.""" + return self.staging_bytes > 0 + + +def staging_size(library: SteamLibrary, appid: int) -> int: + staged = library.steamapps / "downloading" / str(appid) + if not staged.is_dir(): + return 0 + return sum(f.stat().st_size for f in staged.rglob("*") if f.is_file()) + + +def _install_dir(library: SteamLibrary, app_state: dict) -> Path | None: + installdir = app_state.get("installdir") + if not isinstance(installdir, str) or not installdir: + return None + return library.common / installdir + + +def inspect(library: SteamLibrary, appid: int) -> GhostReport: + """Classify `appid` in `library`. An empty `reasons` means it is not a ghost. + + Every condition is required. They are deliberately a conjunction: a real + install fails several of them at once, and a single `or` anywhere in this + chain is the difference between cleaning up a lie and deleting someone's + game. + """ + manifest_path = library.steamapps / f"appmanifest_{appid}.acf" + empty = GhostReport(appid, manifest_path, None, 0, 0, ()) + + if not manifest_path.is_file(): + return empty + try: + app_state = recognize.read_app_state(manifest_path) + except vdf.VdfParseError: + return empty + if str(app_state.get("appid", "")) != str(appid): + return empty + + install_dir = _install_dir(library, app_state) + content_bytes = ( + recognize.measure_content_size(install_dir) + if install_dir is not None and install_dir.is_dir() + else 0 + ) + staged = staging_size(library, appid) + known = GhostReport(appid, manifest_path, install_dir, content_bytes, staged, ()) + + state_flags = app_state.get("StateFlags", "") + if not (str(state_flags).isdigit() and int(state_flags) & INSTALLED_STATE_FLAG): + return known + depots = app_state.get("InstalledDepots") + if depots: + return known + if str(app_state.get("SizeOnDisk", "0")) != "0": + return known + if content_bytes: + return known + # Our own downloader's cache means a real run is in progress against this + # directory; an empty install dir mid-download is not a ghost. + if install_dir is not None and recognize.discover_cached_depot_manifests( + install_dir + ): + return known + + return GhostReport( + appid, + manifest_path, + install_dir, + content_bytes, + staged, + ( + f"manifest claims StateFlags={state_flags} (installed)", + "InstalledDepots is empty", + "SizeOnDisk is 0", + "no game content on disk", + ), + ) + + +def scan(libraries: list[SteamLibrary]) -> list[GhostReport]: + """Every ghost across every library, checked one manifest at a time.""" + found: list[GhostReport] = [] + for library in libraries: + if not library.steamapps.is_dir(): + continue + for manifest in sorted(library.steamapps.glob("appmanifest_*.acf")): + digits = manifest.stem.removeprefix("appmanifest_") + if not digits.isdigit(): + continue + report = inspect(library, int(digits)) + if report.is_ghost: + found.append(report) + return found + + +def repair( + report: GhostReport, logger: logging.Logger, *, now_unix: int | None = None +) -> list[Path]: + """Clear a COLD ghost away. Returns the paths it touched. + + Refuses a warm ghost outright: those bytes belong to `takeover`. + """ + if not report.is_ghost: + raise ValueError(f"appid {report.appid} is not a ghost; refusing to touch it") + if report.is_warm: + raise ValueError( + f"appid {report.appid} has {report.staging_bytes / 1e9:.1f} GB of staged " + "Steam content. That is a partial download, not a ghost to delete -- " + "install it normally so the bytes are reused." + ) + + stamp = int(time.time()) if now_unix is None else now_unix + touched: list[Path] = [] + + # Rename rather than unlink: reversible, atomic, and it leaves the evidence + # on disk if the diagnosis was somehow wrong. + backup = report.manifest_path.with_name( + f"{report.manifest_path.name}.ghost-{stamp}" + ) + report.manifest_path.rename(backup) + logger.info("Moved bogus manifest to %s", backup) + touched.append(backup) + + if report.install_dir is not None and report.install_dir.is_dir(): + try: + # rmdir, never rmtree: the kernel refuses a non-empty directory, so + # a real install cannot be destroyed here even if inspect() were wrong. + report.install_dir.rmdir() + except OSError as exc: + logger.warning("Left %s in place: %s", report.install_dir, exc) + else: + logger.info("Removed empty %s", report.install_dir) + touched.append(report.install_dir) + + return touched diff --git a/_installer_import/steam_installer/resolve.py b/_installer_import/steam_installer/resolve.py index 4e9c7e7..42fed8e 100644 --- a/_installer_import/steam_installer/resolve.py +++ b/_installer_import/steam_installer/resolve.py @@ -1,22 +1,46 @@ -"""Resolve a game name to a Steam AppID via the public store-search endpoint. +"""Resolve a game name to a Steam AppID. -store.steampowered.com is known to TLS-reject scripted clients from at least -some networks (confirmed: even its bare homepage fails the TLS handshake for -plain curl/urllib here, while every other Steam domain works fine -- looks -like storefront-specific bot detection, not a general outage). Treat this as -best-effort: search() can fail, and callers should fall back to asking for a -numeric AppID directly rather than treating a search failure as fatal. +Two endpoints, tried in order, because the obvious one does not work here: + +1. `steamcommunity.com/actions/SearchApps/` -- an undocumented but stable + community-side search. Verified working from this machine; "BeamNG" returns + exactly `284160 BeamNG.drive`. +2. `store.steampowered.com/api/storesearch/` -- the storefront API. It + TLS-rejects scripted clients from at least some networks (confirmed: even + its bare homepage fails the handshake for plain curl/urllib here, while + every other Steam domain works -- storefront-specific bot detection, not an + outage). Kept as a fallback, never relied on. + +`ISteamApps/GetAppList` is deliberately not used: it now answers +`404 Method 'GetAppList' not found in interface 'ISteamApps'`. + +Either way this is best-effort. search() can fail, and callers fall back to +asking for a numeric AppID rather than treating a search failure as fatal. """ from __future__ import annotations import json +import time import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass -SEARCH_URL = "https://store.steampowered.com/api/storesearch/?term={term}&cc=us" +COMMUNITY_SEARCH_URL = "https://steamcommunity.com/actions/SearchApps/{term}" +STORE_SEARCH_URL = "https://store.steampowered.com/api/storesearch/?term={term}&cc=us" + +# urllib's default identifies itself as Python-urllib, which Steam's edge +# sometimes refuses outright. A plain browser UA gets a normal response. +USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) steam-game-installer" + +DEFAULT_TIMEOUT = 15.0 + +# DNS here fails transiently often enough to lose roughly one lookup in three. +# A search that dies on a blip sends the user off to find an AppID by hand for +# no reason, so retry briefly before giving up on a backend. +ATTEMPTS = 3 +RETRY_DELAY = 1.0 @dataclass @@ -30,19 +54,80 @@ class ResolveError(Exception): class SearchUnavailable(Exception): - """The store search endpoint couldn't be reached (network/TLS block, not a code bug).""" + """Neither search endpoint could be reached (network/TLS block, not a code bug).""" -def search(name: str) -> list[SearchResult]: - """Query Steam's store search API for candidate matches. No auth needed.""" - url = SEARCH_URL.format(term=urllib.parse.quote(name)) - try: - with urllib.request.urlopen(url, timeout=15) as response: - payload = json.loads(response.read()) - except (urllib.error.URLError, TimeoutError) as exc: - raise SearchUnavailable(str(exc)) from exc - items = payload.get("items", []) - return [SearchResult(appid=item["id"], name=item["name"]) for item in items] +def _fetch_json(url: str, timeout: float, *, attempts: int = ATTEMPTS) -> object: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + last: Exception = RuntimeError("no attempts made") + for attempt in range(attempts): + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read()) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + last = exc + if attempt + 1 < attempts: + # Looked up on the module, not bound as a default, so tests can + # replace it without waiting out the real delay. + time.sleep(RETRY_DELAY) + raise last + + +def _search_community(term: str, timeout: float) -> list[SearchResult]: + """Community endpoint. Returns a bare JSON list; appids arrive as strings.""" + payload = _fetch_json(COMMUNITY_SEARCH_URL.format(term=term), timeout) + if not isinstance(payload, list): + raise TypeError(f"unexpected community search payload: {type(payload)}") + return [ + SearchResult(appid=int(item["appid"]), name=item["name"]) + for item in payload + if str(item.get("appid", "")).isdigit() + ] + + +def _search_store(term: str, timeout: float) -> list[SearchResult]: + """Storefront endpoint. Returns {"items": [{"id": int, "name": str}, ...]}.""" + payload = _fetch_json(STORE_SEARCH_URL.format(term=term), timeout) + if not isinstance(payload, dict): + raise TypeError(f"unexpected store search payload: {type(payload)}") + return [ + SearchResult(appid=int(item["id"]), name=item["name"]) + for item in payload.get("items", []) + ] + + +def search(name: str, *, timeout: float = DEFAULT_TIMEOUT) -> list[SearchResult]: + """Candidate matches for `name`, newest-working backend first. No auth needed.""" + term = urllib.parse.quote(name) + failures: list[str] = [] + for label, backend in ( + ("steamcommunity", _search_community), + ("store.steampowered", _search_store), + ): + try: + return backend(term, timeout) + except ( + urllib.error.URLError, + TimeoutError, + OSError, + TypeError, + ValueError, + KeyError, + ) as exc: + failures.append(f"{label}: {exc}") + raise SearchUnavailable("; ".join(failures)) + + +def exact_match(results: list[SearchResult], name: str) -> SearchResult | None: + """The one result whose title is exactly `name`, case-insensitively. + + Steam's own ranking puts "Portal 2" above "Portal", so preferring an exact + title is the difference between typing a full name and getting the sequel. + Ambiguous only in theory: if two apps share a title we refuse to guess. + """ + wanted = name.strip().casefold() + hits = [result for result in results if result.name.strip().casefold() == wanted] + return hits[0] if len(hits) == 1 else None def prompt_confirm(results: list[SearchResult]) -> SearchResult: diff --git a/_installer_import/steam_installer/steamctl.py b/_installer_import/steam_installer/steamctl.py new file mode 100644 index 0000000..4ee4572 --- /dev/null +++ b/_installer_import/steam_installer/steamctl.py @@ -0,0 +1,72 @@ +"""Stop and start the Steam client around operations it would otherwise fight. + +Steam owns `appmanifest_*.acf` while it runs: it rewrites them from its own +in-memory state, so anything this tool writes underneath a live client is +discarded. It also actively writes into `steamapps/downloading//`, so +adopting a staged tree while it runs races a live download. + +The client is only ever asked to leave politely (`steam -shutdown`). Nothing +here sends a signal: SIGKILLing a client mid-write is how a manifest ends up +truncated, and a hung Steam is a reason to stop and tell the user, not to +escalate. +""" + +from __future__ import annotations + +import logging +import subprocess +import time +from collections.abc import Callable + +from steam_installer import recognize + +SHUTDOWN_TIMEOUT = 60.0 +POLL_INTERVAL = 0.5 + + +class SteamStillRunning(Exception): + """Steam was asked to exit and did not.""" + + +def stop( + logger: logging.Logger, + *, + timeout: float = SHUTDOWN_TIMEOUT, + sleep: Callable[[float], None] = time.sleep, +) -> bool: + """Shut Steam down. Returns True if it had been running, False if it wasn't. + + Raises SteamStillRunning if it refuses to exit, so callers can abort + without having touched anything. + """ + if not recognize.steam_client_pids(): + return False + + logger.info("Shutting down the Steam client...") + subprocess.run(["steam", "-shutdown"], check=False, capture_output=True) + + waited = 0.0 + while waited < timeout: + if not recognize.steam_client_pids(): + logger.info("Steam exited after %.0fs.", waited) + return True + sleep(POLL_INTERVAL) + waited += POLL_INTERVAL + + pids = recognize.steam_client_pids() + raise SteamStillRunning( + f"Steam is still running after {timeout:.0f}s (pid " + f"{', '.join(str(p) for p in pids)}). Close it by hand and re-run." + ) + + +def start(logger: logging.Logger) -> None: + """Relaunch Steam, detached, so it outlives this process.""" + logger.info("Restarting the Steam client...") + # Detached on purpose: Steam outlives this process and is never waited on. + subprocess.Popen( + ["steam"], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) diff --git a/_installer_import/steam_installer/takeover.py b/_installer_import/steam_installer/takeover.py new file mode 100644 index 0000000..9ea9bea --- /dev/null +++ b/_installer_import/steam_installer/takeover.py @@ -0,0 +1,118 @@ +"""Adopt content the Steam client already downloaded instead of refetching it. + +Steam stages an in-progress download into `steamapps/downloading//`, +laid out exactly like the finished game (confirmed on a live BeamNG.drive +staging tree: `Bin64/`, `BinLinux/`, `campaigns/`, `content/`), and commits it +into `steamapps/common//` when it finishes. + +This tool exists to make installs faster. Throwing away tens of gigabytes the +user already waited for, to start again from zero, would make it slower. So +when a staged tree exists, hardlink it into the install directory first and +let depotdownloader do the rest. + +That works because depotdownloader verifies what is already on disk before +fetching: `ContentDownloader.cs` runs `Util.ValidateSteam3FileChecksums()` on +an existing file and skips it entirely when `neededChunks.Count == 0`. It +needs no prior `.DepotDownloader` state to do this, so adopted files count. + +Hardlinks, not moves, and not copies. The staged tree keeps its entries until +`finalize.finish` clears it at the very end, so a failed run still leaves Steam +able to resume its own download -- which a move would destroy. Linking costs no +extra disk, where a copy of a 47 GB tree would need 47 GB. + +The two trees share inodes, so depotdownloader patching a partial file in place +(it seeks and writes into the existing file rather than replacing it) rewrites +Steam's copy too. That is not a hazard, it is the point: both names converge on +the same corrected content, so whichever downloader finishes the job, the other +one's view is already right. Steam chunk-validates on resume regardless. + +This does mean the staged bytes are not a pristine backup. Nothing is deleted, +but they are not frozen either -- on ext4 there is no reflink to get both. +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +from steam_installer.library import SteamLibrary + + +@dataclass(frozen=True) +class Staged: + appid: int + staging_dir: Path + install_dir: Path + file_count: int + bytes_staged: int + + +@dataclass(frozen=True) +class Adoption: + linked: int + skipped: int + failed: int + bytes_adopted: int + + +def find_staged(library: SteamLibrary, appid: int, install_dir: Path) -> Staged | None: + """Steam's in-progress download for `appid`, or None if there isn't one.""" + staging_dir = library.steamapps / "downloading" / str(appid) + if not staging_dir.is_dir(): + return None + files = [f for f in staging_dir.rglob("*") if f.is_file() and not f.is_symlink()] + if not files: + return None + return Staged( + appid=appid, + staging_dir=staging_dir, + install_dir=install_dir, + file_count=len(files), + bytes_staged=sum(f.stat().st_size for f in files), + ) + + +def adopt(staged: Staged, logger: logging.Logger) -> Adoption: + """Hardlink the staged tree into the install directory. Originals survive. + + Never overwrites a file that is already in place: an existing file is + either content this tool put there or content Steam committed, and + depotdownloader will hash-check both anyway. + """ + linked = skipped = failed = 0 + adopted_bytes = 0 + + for source in sorted(staged.staging_dir.rglob("*")): + if not source.is_file() or source.is_symlink(): + continue + relative = source.relative_to(staged.staging_dir) + destination = staged.install_dir / relative + if destination.exists(): + skipped += 1 + continue + destination.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, destination) + except OSError as exc: + # Not fatal: depotdownloader simply downloads whatever we failed to + # adopt. Losing a link is slower, never wrong. + if failed == 0: + logger.warning("Could not hardlink %s: %s", relative, exc) + failed += 1 + else: + linked += 1 + adopted_bytes += source.stat().st_size + + if failed: + logger.warning("%d staged file(s) could not be adopted.", failed) + logger.info( + "Adopted %.2f GB from Steam's staging tree (%d linked, %d already present).", + adopted_bytes / 1e9, + linked, + skipped, + ) + return Adoption( + linked=linked, skipped=skipped, failed=failed, bytes_adopted=adopted_bytes + ) diff --git a/_installer_import/steam_installer/target.py b/_installer_import/steam_installer/target.py new file mode 100644 index 0000000..6d956ba --- /dev/null +++ b/_installer_import/steam_installer/target.py @@ -0,0 +1,112 @@ +"""Work out which Steam app the user meant from one free-form argument. + +Four accepted forms, all resolving to the same app: + + 284160 + https://steamcommunity.com/app/284160/guides/ + https://store.steampowered.com/app/284160/BeamNGdrive/ + BeamNG.drive + BeamNG + +Whatever the input, the appid is always confirmed through steamcmd's own +`app_info_print` before anything acts on it, so the name and installdir used +downstream come from Steam rather than from a search index that ranks +"Portal 2" above "Portal". +""" + +from __future__ import annotations + +import re +import sys +from dataclasses import dataclass + +from steam_installer import appinfo, resolve + +# `app/` preceded by a delimiter: matches both store and community +# URLs, any trailing path or query, and `steam://` links. Not anchored to a +# scheme, so a bare "steamcommunity.com/app/284160" works too. +APP_URL_RE = re.compile(r"(?:^|[/?&])app/(\d+)(?:[/?#]|$)") + + +class TargetError(Exception): + """The input could not be resolved to exactly one Steam app.""" + + +@dataclass(frozen=True) +class Resolved: + metadata: appinfo.AppMetadata + source: str # "flag" | "appid" | "url" | "exact" | "picked" + + +def parse_literal(text: str) -> tuple[int, str] | None: + """(appid, source) when the text literally names one, else None.""" + stripped = text.strip() + if stripped.isdigit() and int(stripped) > 0: + return int(stripped), "appid" + match = APP_URL_RE.search(stripped) + if match: + return int(match.group(1)), "url" + return None + + +def _confirm(appid: int, source: str) -> Resolved: + try: + return Resolved(appinfo.get_metadata(appid), source) + except appinfo.AppInfoError as exc: + raise TargetError(f"appid {appid} is not a known Steam app ({exc})") from exc + + +def _by_name(text: str, *, interactive: bool) -> Resolved: + try: + results = resolve.search(text) + except resolve.SearchUnavailable as exc: + raise TargetError( + f"Steam search is unavailable ({exc}). Pass the numeric AppID instead." + ) from exc + + if not results: + raise TargetError(f"No Steam app matches {text!r}.") + if len(results) == 1: + return _confirm(results[0].appid, "only match") + + # Exact title first: Steam ranks "Portal 2" above "Portal", so without this + # a fully-typed name silently installs the sequel. + exact = resolve.exact_match(results, text) + if exact is not None: + return _confirm(exact.appid, "exact") + + if not interactive: + # Fail closed. Guessing here is how the wrong 90 GB gets downloaded. + print(f"{text!r} is ambiguous. Candidates:", file=sys.stderr) + for result in results: + print(f" {result.appid} {result.name}", file=sys.stderr) + raise TargetError( + "Ambiguous name in a non-interactive run. Re-run with the exact " + "title or the numeric AppID." + ) + + return _confirm(resolve.prompt_confirm(results).appid, "picked") + + +def resolve_target( + text: str | None, appid: int | None, *, interactive: bool +) -> Resolved: + """Resolve --appid, or a free-form target, to confirmed app metadata.""" + if appid is not None: + return _confirm(appid, "flag") + if text is None: + raise TargetError("No target given.") + + literal = parse_literal(text) + if literal is not None: + candidate, source = literal + try: + return _confirm(candidate, source) + except TargetError: + # Digits are not unambiguous -- a game literally called "2048" + # exists -- so an unknown appid falls through to a name search + # rather than dead-ending. A URL cannot be a name, so it doesn't. + if source != "appid": + raise + + return _by_name(text, interactive=interactive) diff --git a/_installer_import/tests/conftest.py b/_installer_import/tests/conftest.py index fbc4176..3757bdf 100644 --- a/_installer_import/tests/conftest.py +++ b/_installer_import/tests/conftest.py @@ -1,15 +1,32 @@ -"""Shared test setup. +"""Shared test setup and fixtures. `test_module_is_runnable_as_a_script` runs `python -m steam_installer.cli` in a subprocess. Without the hook below, coverage collected in that child is thrown away and the `__main__` guard looks uncovered. + +The Steam-tree fixtures live here rather than in one test module because +several files need them, and importing a fixture across test modules confuses +both pytest and the linter. """ +import logging import os import pathlib import pytest +from steam_installer import ( + appinfo, + cli, + config, + depotdl, + deps, + recognize, + runlog, + vdf, +) +from steam_installer.library import SteamLibrary + @pytest.fixture def coverage_subprocess_env(): @@ -19,3 +36,98 @@ def coverage_subprocess_env(): env["COVERAGE_PROCESS_START"] = str(root / "pyproject.toml") env["PYTHONPATH"] = str(root) + os.pathsep + env.get("PYTHONPATH", "") return env + + +GHOST_ACF = """"AppState" +{ +\t"appid"\t\t"284160" +\t"universe"\t\t"1" +\t"name"\t\t"BeamNG.drive" +\t"StateFlags"\t\t"4" +\t"installdir"\t\t"BeamNG.drive" +\t"LastUpdated"\t\t"0" +\t"SizeOnDisk"\t\t"0" +\t"StagingSize"\t\t"0" +\t"buildid"\t\t"24617469" +\t"DownloadType"\t\t"1" +\t"BytesToDownload"\t\t"0" +\t"BytesDownloaded"\t\t"0" +\t"TargetBuildID"\t\t"24617469" +\t"InstalledDepots" +\t{ +\t} +\t"UserConfig" +\t{ +\t\t"language"\t\t"english" +\t} +} +""" + + +@pytest.fixture +def library(tmp_path): + lib = SteamLibrary(tmp_path / "Steam") + lib.steamapps.mkdir(parents=True) + lib.common.mkdir(parents=True) + return lib + + +def _write(library, appid, text): + path = library.steamapps / f"appmanifest_{appid}.acf" + path.write_text(text) + return path + + +def _real_install(library, appid=475150, installdir="RealGame", size="13908166765"): + """A manifest shaped like the genuinely installed apps in a live library.""" + _write( + library, + appid, + vdf.dumps( + "AppState", + { + "appid": str(appid), + "StateFlags": "4", + "installdir": installdir, + "SizeOnDisk": size, + "InstalledDepots": {"475151": {"manifest": "123"}}, + }, + ), + ) + content = library.common / installdir + content.mkdir(parents=True, exist_ok=True) + (content / "game.pak").write_bytes(b"x" * 4096) + return content + + +@pytest.fixture +def isolated_cli(tmp_path, monkeypatch): + monkeypatch.setattr(runlog, "LOG_DIR", tmp_path / "state") + monkeypatch.setattr(config, "CONFIG_DIR", tmp_path / "cfg") + monkeypatch.setattr(config, "CONFIG_PATH", tmp_path / "cfg" / "config") + monkeypatch.delenv(config.ENV_VAR, raising=False) + monkeypatch.setattr(deps.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr(recognize, "steam_client_pids", list) + yield + logger = logging.getLogger(runlog.LOGGER_NAME) + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + +@pytest.fixture +def cli_steam(tmp_path, monkeypatch, isolated_cli): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + library.common.mkdir(parents=True) + monkeypatch.setattr(cli, "find_steam_root", lambda: library.path) + monkeypatch.setattr(cli, "list_libraries", lambda root: [library]) + monkeypatch.setattr( + appinfo, + "get_metadata", + lambda appid: appinfo.AppMetadata(appid, "BeamNG.drive", "BeamNG.drive"), + ) + monkeypatch.setattr( + depotdl, "download", lambda **kw: pytest.fail("must not download") + ) + return library diff --git a/_installer_import/tests/test_cli.py b/_installer_import/tests/test_cli.py index c460dfd..db0aef3 100644 --- a/_installer_import/tests/test_cli.py +++ b/_installer_import/tests/test_cli.py @@ -12,7 +12,7 @@ import sys import pytest -from steam_installer import appinfo, cli, recognize, resolve, runlog, vdf +from steam_installer import appinfo, cli, depotdl, deps, recognize, runlog, vdf from steam_installer.library import SteamLibrary @@ -53,21 +53,23 @@ def steam(tmp_path, monkeypatch): monkeypatch.setattr(cli, "find_steam_root", lambda: root) monkeypatch.setattr(cli, "list_libraries", lambda r: [library]) monkeypatch.setattr(cli, "pick_install_target", lambda libs, d: install_dir) - monkeypatch.setattr(cli.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr(deps.shutil, "which", lambda name: f"/usr/bin/{name}") monkeypatch.setattr( - cli.appinfo, + appinfo, "get_metadata", lambda appid: appinfo.AppMetadata(appid, "KCD2", "KingdomComeDeliverance2"), ) monkeypatch.setattr( - cli.appinfo, + appinfo, "get_depot_plan", lambda appid, language: appinfo.DepotPlan({"1771303"}, {"228989"}), ) - monkeypatch.setattr(cli.appinfo, "get_build_id", lambda appid: "23914554") + monkeypatch.setattr(appinfo, "get_build_id", lambda appid: "23914554") + monkeypatch.setattr(recognize, "steam_client_pids", lambda: []) monkeypatch.setattr(recognize, "steam_client_pids", lambda: []) - monkeypatch.setattr(cli.recognize, "steam_client_pids", lambda: []) monkeypatch.setattr("builtins.input", lambda *a: "y") + # Prompting is tty-gated now, and pytest's stdin is not a tty. + monkeypatch.setattr(cli, "_is_interactive", lambda args: True) return library, install_dir @@ -75,12 +77,12 @@ ARGV = ["--appid", "1771300", "--username", "u", "--language", "english"] def _stub_download(monkeypatch, returncode=0): - monkeypatch.setattr(cli.depotdl, "download", lambda **kw: returncode) + monkeypatch.setattr(depotdl, "download", lambda **kw: returncode) def _stub_steamcmd(monkeypatch, returncode, crashed=False): monkeypatch.setattr( - cli.recognize, + recognize, "run_steamcmd_validate", lambda appid, username, logger: recognize.ValidateResult( completed=returncode == 0, @@ -112,7 +114,7 @@ def test_refuses_when_a_required_depot_never_downloaded(steam, monkeypatch): _stub_download(monkeypatch) _stub_steamcmd(monkeypatch, returncode=8) monkeypatch.setattr( - cli.appinfo, + appinfo, "get_depot_plan", lambda appid, language: appinfo.DepotPlan({"1771303", "9999999"}, set()), ) @@ -133,13 +135,15 @@ def test_refuses_when_depotdownloader_fails(steam, monkeypatch): assert app_state["StateFlags"] == "1026" -def test_refuses_before_downloading_when_steam_is_running(steam, monkeypatch): - """Steam owns the manifest while it runs; refuse up front rather than after - an hour of downloading.""" - library, _ = steam - monkeypatch.setattr(cli.recognize, "steam_client_pids", lambda: [4242]) +def test_declining_to_close_steam_aborts_before_downloading(steam, monkeypatch): + """Steam owns the manifest while it runs. The tool offers to close it, but a + "no" must stop the run rather than download for an hour and lose the write.""" + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [4242]) + monkeypatch.setattr( + "builtins.input", lambda *a: "n" if "Steam is running" in a[0] else "y" + ) called = [] - monkeypatch.setattr(cli.depotdl, "download", lambda **kw: called.append(1) or 0) + monkeypatch.setattr(depotdl, "download", lambda **kw: called.append(1) or 0) assert cli.main(ARGV) == 1 assert called == [] @@ -165,10 +169,10 @@ def test_clean_steamcmd_run_short_circuits(steam, monkeypatch): ) return recognize.ValidateResult(True, False, 0) - monkeypatch.setattr(cli.recognize, "run_steamcmd_validate", fake_validate) + monkeypatch.setattr(recognize, "run_steamcmd_validate", fake_validate) called = [] monkeypatch.setattr( - cli.recognize, + recognize, "finish_manifest_manually", lambda **kw: called.append(1), ) @@ -205,10 +209,10 @@ def test_a_complete_install_after_a_failed_steamcmd_is_not_credited_to_it( ) return recognize.ValidateResult(True, False, 5) - monkeypatch.setattr(cli.recognize, "run_steamcmd_validate", fake_validate) + monkeypatch.setattr(recognize, "run_steamcmd_validate", fake_validate) called = [] monkeypatch.setattr( - cli.recognize, "finish_manifest_manually", lambda **kw: called.append(1) + recognize, "finish_manifest_manually", lambda **kw: called.append(1) ) assert cli.main(ARGV) == 0 @@ -221,7 +225,7 @@ def test_a_complete_install_after_a_failed_steamcmd_is_not_credited_to_it( def test_user_declining_confirmation_aborts(steam, monkeypatch): monkeypatch.setattr("builtins.input", lambda *a: "n") called = [] - monkeypatch.setattr(cli.depotdl, "download", lambda **kw: called.append(1) or 0) + monkeypatch.setattr(depotdl, "download", lambda **kw: called.append(1) or 0) assert cli.main(ARGV) == 1 assert called == [] @@ -230,7 +234,7 @@ def test_unhandled_error_returns_nonzero_and_names_the_log(steam, monkeypatch, c def boom(**kwargs): raise RuntimeError("depotdownloader exploded") - monkeypatch.setattr(cli.depotdl, "download", boom) + monkeypatch.setattr(depotdl, "download", boom) assert cli.main(ARGV) == 1 assert "Run log" in capsys.readouterr().out @@ -268,43 +272,11 @@ def test_missing_manifest_with_steamcmd_blames_steamcmd(steam, monkeypatch, caps assert "steamcmd never ran far enough" in capsys.readouterr().out -def test_require_tool_exits_when_binary_missing(monkeypatch, capsys): - monkeypatch.setattr(cli.shutil, "which", lambda name: None) - with pytest.raises(SystemExit): - cli._require_tool("depotdownloader", "See ~/depotdownloader.") - assert "not found on PATH" in capsys.readouterr().err - - -def test_resolve_appid_uses_search_when_no_appid(monkeypatch): - monkeypatch.setattr( - cli.resolve, "search", lambda name: [resolve.SearchResult(1771300, "KCD2")] - ) - monkeypatch.setattr(cli.resolve, "prompt_confirm", lambda results: results[0]) - - args = cli._build_parser().parse_args(["kingdom", "--username", "u"]) - assert cli._resolve_appid(args) == 1771300 - - -def test_resolve_appid_exits_when_search_unavailable(monkeypatch, capsys): - """store.steampowered.com blocks scripted clients on some networks; tell the - user to pass --appid rather than dying with a traceback.""" - - def boom(name): - raise resolve.SearchUnavailable("tls blocked") - - monkeypatch.setattr(cli.resolve, "search", boom) - args = cli._build_parser().parse_args(["kingdom", "--username", "u"]) - - with pytest.raises(SystemExit): - cli._resolve_appid(args) - assert "pass --appid" in capsys.readouterr().err - - def test_language_defaults_to_english_when_prompt_is_empty(steam, monkeypatch): seen = {} monkeypatch.setattr("builtins.input", lambda *a: "y" if "Proceed" in a[0] else "") monkeypatch.setattr( - cli.appinfo, + appinfo, "get_depot_plan", lambda appid, language: ( seen.setdefault("language", language) @@ -328,8 +300,10 @@ def test_reports_the_known_install_script_crash(steam, monkeypatch, capsys): assert "known install-script crash" in capsys.readouterr().out -def test_reports_cleared_staging(steam, monkeypatch, capsys): - library, _ = steam +def test_adopts_staged_content_then_clears_it(steam, monkeypatch, capsys): + """Steam's staging tree is adopted into the install dir before downloading, + and only cleared once the install is finished -- never discarded up front.""" + library, install_dir = steam staged = library.steamapps / "downloading" / "1771300" staged.mkdir(parents=True) (staged / "big.pak").write_bytes(b"x" * 2_000_000) @@ -337,7 +311,12 @@ def test_reports_cleared_staging(steam, monkeypatch, capsys): _stub_steamcmd(monkeypatch, returncode=8) assert cli.main(ARGV) == 0 - assert "stale Steam staging data" in capsys.readouterr().out + + out = capsys.readouterr().out + assert "Hardlinking it" in out + assert "now-redundant Steam staging data" in out + # Adopted, not copied: the file is one inode with two names. + assert (install_dir / "big.pak").stat().st_size == 2_000_000 def test_fails_when_written_manifest_does_not_read_back_as_installed( @@ -348,7 +327,7 @@ def test_fails_when_written_manifest_does_not_read_back_as_installed( _stub_download(monkeypatch) _stub_steamcmd(monkeypatch, returncode=8) monkeypatch.setattr( - cli.recognize, "manifest_reflects_real_install", lambda *a, **kw: False + recognize, "manifest_reflects_real_install", lambda *a, **kw: False ) assert cli.main(ARGV) == 1 diff --git a/_installer_import/tests/test_cli_modes.py b/_installer_import/tests/test_cli_modes.py new file mode 100644 index 0000000..aecbb47 --- /dev/null +++ b/_installer_import/tests/test_cli_modes.py @@ -0,0 +1,51 @@ +"""The CLI front door: turning one free-form argument into a resolved target. + +test_cli.py covers the download/recognize decision logic; test_cli_steam.py +covers ghosts, usernames and the Steam client. This file covers detection and +the modes that deliberately change nothing. +""" + +from steam_installer import cli, deps, target +from tests.conftest import GHOST_ACF + + +def test_a_bare_appid_resolves_and_reports_without_changing_anything(cli_steam, capsys): + assert cli.main(["--dry-run", "284160"]) == 0 + out = capsys.readouterr().out + assert "BeamNG.drive (appid 284160, matched by appid)" in out + assert "nothing was changed" in out + + +def test_a_steam_url_resolves(cli_steam, capsys): + assert cli.main(["--dry-run", "https://steamcommunity.com/app/284160/guides/"]) == 0 + assert "matched by url" in capsys.readouterr().out + + +def test_a_dry_run_reports_staged_bytes_it_would_adopt(cli_steam, capsys): + staged = cli_steam.steamapps / "downloading" / "284160" + staged.mkdir(parents=True) + (staged / "part.pak").write_bytes(b"x" * 4096) + + assert cli.main(["--dry-run", "284160"]) == 0 + assert "would be adopted, not re-downloaded" in capsys.readouterr().out + + +def test_a_dry_run_names_a_ghost_manifest(cli_steam, capsys): + (cli_steam.steamapps / "appmanifest_284160.acf").write_text(GHOST_ACF) + + assert cli.main(["--dry-run", "284160"]) == 0 + assert "GHOST manifest" in capsys.readouterr().out + + +def test_an_ambiguous_name_exits_two(cli_steam, monkeypatch): + def boom(text, appid, *, interactive): + raise target.TargetError("Ambiguous name") + + monkeypatch.setattr(target, "resolve_target", boom) + assert cli.main(["--dry-run", "witcher"]) == 2 + + +def test_a_missing_dependency_exits_one(cli_steam, monkeypatch, capsys): + monkeypatch.setattr(deps.shutil, "which", lambda name: None) + assert cli.main(["--dry-run", "284160"]) == 1 + assert "depotdownloader" in capsys.readouterr().out diff --git a/_installer_import/tests/test_cli_steam.py b/_installer_import/tests/test_cli_steam.py new file mode 100644 index 0000000..5ee916a --- /dev/null +++ b/_installer_import/tests/test_cli_steam.py @@ -0,0 +1,205 @@ +"""What the CLI does around a real install: ghosts, usernames, the Steam client. + +Split from test_cli_modes.py to keep both files under the repo's 250-line cap. +""" + +import pytest + +from steam_installer import appinfo, cli, config, depotdl, ghost, recognize +from tests.conftest import GHOST_ACF + + +def test_a_cold_ghost_blocks_an_install_and_names_the_fix(cli_steam, capsys): + """Installing over a fabricated manifest is how Steam was tricked into + re-downloading in the first place; stop and say what to run.""" + (cli_steam.steamapps / "appmanifest_284160.acf").write_text(GHOST_ACF) + + assert cli.main(["--yes", "284160", "--username", "u"]) == 1 + assert "./run.sh --repair-ghosts" in capsys.readouterr().out + + +def test_a_warm_ghost_does_not_block_an_install(cli_steam, monkeypatch): + """Staged bytes are the whole reason to run this tool, not a blocker.""" + (cli_steam.steamapps / "appmanifest_284160.acf").write_text(GHOST_ACF) + staged = cli_steam.steamapps / "downloading" / "284160" + staged.mkdir(parents=True) + (staged / "part.pak").write_bytes(b"x" * 4096) + reached = [] + monkeypatch.setattr(depotdl, "download", lambda **kw: reached.append(1) or 1) + monkeypatch.setattr( + appinfo, "get_depot_plan", lambda a, l: appinfo.DepotPlan(set(), set()) + ) + monkeypatch.setattr(appinfo, "get_build_id", lambda a: "1") + + cli.main(["--yes", "284160", "--username", "u"]) + + assert reached == [1] + + +def test_repair_ghosts_clears_a_cold_ghost(cli_steam, capsys): + (cli_steam.steamapps / "appmanifest_284160.acf").write_text(GHOST_ACF) + + assert cli.main(["--repair-ghosts"]) == 0 + + assert not (cli_steam.steamapps / "appmanifest_284160.acf").exists() + assert "Ghost: appid 284160" in capsys.readouterr().out + + +def test_repair_ghosts_refuses_to_discard_a_warm_ghosts_download(cli_steam, capsys): + (cli_steam.steamapps / "appmanifest_284160.acf").write_text(GHOST_ACF) + staged = cli_steam.steamapps / "downloading" / "284160" + staged.mkdir(parents=True) + (staged / "part.pak").write_bytes(b"x" * 4096) + + assert cli.main(["--repair-ghosts"]) == 0 + + assert (cli_steam.steamapps / "appmanifest_284160.acf").exists() + assert (staged / "part.pak").exists() + assert "nothing is deleted" in capsys.readouterr().out + + +def test_repair_ghosts_says_so_when_there_is_nothing_to_do(cli_steam, capsys): + assert cli.main(["--repair-ghosts"]) == 0 + assert "No ghost installs found" in capsys.readouterr().out + + +def test_repair_ghosts_needs_no_target(cli_steam): + """It is the one mode that runs without naming a game.""" + assert cli.main(["--repair-ghosts"]) == 0 + + +def test_a_saved_username_removes_the_need_for_the_flag(cli_steam, monkeypatch): + config.save_username("saved-user") + seen = {} + monkeypatch.setattr( + depotdl, + "download", + lambda **kw: seen.setdefault("username", kw["username"]) or 1, + ) + monkeypatch.setattr( + appinfo, "get_depot_plan", lambda a, l: appinfo.DepotPlan(set(), set()) + ) + monkeypatch.setattr(appinfo, "get_build_id", lambda a: "1") + + cli.main(["--yes", "284160"]) + + assert seen["username"] == "saved-user" + + +def test_no_username_anywhere_fails_with_an_explanation(cli_steam, capsys): + assert cli.main(["--yes", "284160"]) == 1 + assert "No Steam username available" in capsys.readouterr().out + + +def test_a_non_interactive_run_without_yes_does_not_start_a_download(cli_steam, capsys): + """A piped invocation never asked for a 50 GB download; refuse rather than + assume.""" + assert cli.main(["284160", "--username", "u"]) == 1 + assert "use --yes" in capsys.readouterr().out + + +def test_language_defaults_to_english_when_not_interactive(cli_steam, monkeypatch): + seen = {} + monkeypatch.setattr( + appinfo, + "get_depot_plan", + lambda a, language: ( + seen.setdefault("language", language) or appinfo.DepotPlan(set(), set()) + ), + ) + monkeypatch.setattr(appinfo, "get_build_id", lambda a: "1") + monkeypatch.setattr(depotdl, "download", lambda **kw: 1) + + cli.main(["--yes", "284160", "--username", "u"]) + + assert seen["language"] == "english" + + +def test_nothing_to_do_at_all_is_a_usage_error(): + with pytest.raises(SystemExit): + cli.main(["--username", "u"]) + + +def test_a_ghost_for_a_different_app_does_not_block_this_one(cli_steam, monkeypatch): + (cli_steam.steamapps / "appmanifest_999.acf").write_text( + GHOST_ACF.replace('"appid"\t\t"284160"', '"appid"\t\t"999"') + ) + monkeypatch.setattr(depotdl, "download", lambda **kw: 1) + monkeypatch.setattr( + appinfo, "get_depot_plan", lambda a, l: appinfo.DepotPlan(set(), set()) + ) + monkeypatch.setattr(appinfo, "get_build_id", lambda a: "1") + + assert cli.main(["--yes", "284160", "--username", "u"]) == 1 + + +def _installable(monkeypatch, library): + monkeypatch.setattr( + appinfo, "get_depot_plan", lambda a, l: appinfo.DepotPlan(set(), set()) + ) + monkeypatch.setattr(appinfo, "get_build_id", lambda a: "1") + monkeypatch.setattr(depotdl, "download", lambda **kw: 1) + + +def test_declining_the_shutdown_prompt_stops_the_run(cli_steam, monkeypatch): + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [42]) + monkeypatch.setattr(cli, "_is_interactive", lambda args: True) + monkeypatch.setattr( + "builtins.input", lambda *a: "n" if "Steam is running" in a[0] else "y" + ) + monkeypatch.setattr( + depotdl, "download", lambda **kw: pytest.fail("must not download") + ) + + assert cli.main(["284160", "--username", "u", "--language", "english"]) == 1 + + +def test_steam_is_closed_and_reopened_around_the_install(cli_steam, monkeypatch): + """It was running before, so it is running after -- whatever the outcome.""" + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [42]) + monkeypatch.setattr(ghost.recognize, "steam_client_pids", lambda: [42]) + events = [] + monkeypatch.setattr( + cli.flow.steamctl, "stop", lambda logger: events.append("stop") or True + ) + monkeypatch.setattr( + cli.flow.steamctl, "start", lambda logger: events.append("start") + ) + _installable(monkeypatch, cli_steam) + + assert cli.main(["--yes", "284160", "--username", "u"]) == 1 + assert events == ["stop", "start"] + + +def test_a_steam_that_refuses_to_exit_aborts_the_run(cli_steam, monkeypatch, capsys): + """Never escalate to a signal: a hung client is the user's to deal with.""" + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [42]) + + def refuse(logger): + raise cli.flow.steamctl.SteamStillRunning("Steam is still running after 60s") + + monkeypatch.setattr(cli.flow.steamctl, "stop", refuse) + monkeypatch.setattr( + depotdl, "download", lambda **kw: pytest.fail("must not download") + ) + + assert cli.main(["--yes", "284160", "--username", "u"]) == 1 + assert "still running" in capsys.readouterr().out + + +def test_accepting_the_shutdown_prompt_closes_and_reopens_steam(cli_steam, monkeypatch): + """The interactive path: asked, answered yes, Steam cycled around the run.""" + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [42]) + monkeypatch.setattr(cli, "_is_interactive", lambda args: True) + monkeypatch.setattr("builtins.input", lambda *a: "y") + events = [] + monkeypatch.setattr( + cli.flow.steamctl, "stop", lambda logger: events.append("stop") or True + ) + monkeypatch.setattr( + cli.flow.steamctl, "start", lambda logger: events.append("start") + ) + _installable(monkeypatch, cli_steam) + + assert cli.main(["284160", "--username", "u", "--language", "english"]) == 1 + assert events == ["stop", "start"] diff --git a/_installer_import/tests/test_config.py b/_installer_import/tests/test_config.py new file mode 100644 index 0000000..2fd0417 --- /dev/null +++ b/_installer_import/tests/test_config.py @@ -0,0 +1,95 @@ +"""Username persistence and its precedence order.""" + +import pytest + +from steam_installer import config + + +@pytest.fixture(autouse=True) +def isolated_config(tmp_path, monkeypatch): + """Never read or write the real ~/.config during a test run.""" + monkeypatch.setattr(config, "CONFIG_DIR", tmp_path / "cfg") + monkeypatch.setattr(config, "CONFIG_PATH", tmp_path / "cfg" / "config") + monkeypatch.delenv(config.ENV_VAR, raising=False) + + +def test_explicit_username_wins_over_everything(monkeypatch): + monkeypatch.setenv(config.ENV_VAR, "from_env") + config.save_username("from_file") + assert config.resolve_username("explicit", interactive=False) == "explicit" + + +def test_env_wins_over_the_saved_file(monkeypatch): + monkeypatch.setenv(config.ENV_VAR, "from_env") + config.save_username("from_file") + assert config.resolve_username(None, interactive=False) == "from_env" + + +def test_a_blank_env_var_does_not_shadow_the_saved_file(monkeypatch): + monkeypatch.setenv(config.ENV_VAR, " ") + config.save_username("from_file") + assert config.resolve_username(None, interactive=False) == "from_file" + + +def test_the_saved_file_is_used_when_nothing_else_is_given(): + config.save_username("saved") + assert config.resolve_username(None, interactive=False) == "saved" + + +def test_saving_creates_the_directory_and_locks_the_file_down(): + config.save_username("someone") + assert config.CONFIG_PATH.read_text() == "username=someone\n" + assert config.CONFIG_PATH.stat().st_mode & 0o777 == 0o600 + + +def test_saving_twice_replaces_rather_than_appends(): + config.save_username("first") + config.save_username("second") + assert config.load_username() == "second" + + +def test_load_returns_none_when_nothing_is_saved(): + assert config.load_username() is None + + +def test_load_ignores_other_keys_and_malformed_lines(): + config.CONFIG_DIR.mkdir(parents=True) + config.CONFIG_PATH.write_text("# a comment\nnot-a-pair\nother=x\nusername=real\n") + assert config.load_username() == "real" + + +def test_an_empty_username_value_reads_as_unset(): + config.CONFIG_DIR.mkdir(parents=True) + config.CONFIG_PATH.write_text("username=\n") + assert config.load_username() is None + + +def test_prompting_saves_the_answer(monkeypatch): + monkeypatch.setattr("builtins.input", lambda *a: " typed ") + assert config.resolve_username(None, interactive=True) == "typed" + assert config.load_username() == "typed" + + +def test_an_empty_answer_is_refused(monkeypatch): + monkeypatch.setattr("builtins.input", lambda *a: "") + with pytest.raises(config.ConfigError, match="No Steam username entered"): + config.resolve_username(None, interactive=True) + + +def test_non_interactive_with_nothing_available_explains_the_options(): + with pytest.raises(config.ConfigError, match="--username"): + config.resolve_username(None, interactive=False) + + +def test_a_line_with_no_separator_is_skipped_before_a_valid_one(): + """`partition` returns an empty separator for a bare line; that must not be + mistaken for a key with an empty value.""" + config.CONFIG_DIR.mkdir(parents=True) + config.CONFIG_PATH.write_text("username\nusername=real\n") + assert config.load_username() == "real" + + +def test_a_config_file_with_no_username_key_reads_as_unset(): + config.CONFIG_DIR.mkdir(parents=True) + config.CONFIG_PATH.write_text("other=x\n# nothing relevant here\n") + assert config.load_username() is None diff --git a/_installer_import/tests/test_deps.py b/_installer_import/tests/test_deps.py new file mode 100644 index 0000000..1041c7b --- /dev/null +++ b/_installer_import/tests/test_deps.py @@ -0,0 +1,31 @@ +"""The external-binary check. Moved here from test_cli.py with the code it covers.""" + +import pytest + +from steam_installer import deps + + +def test_nothing_missing_when_both_tools_are_on_path(monkeypatch): + monkeypatch.setattr(deps.shutil, "which", lambda name: f"/usr/bin/{name}") + assert deps.missing_tools(skip_steamcmd=False) == [] + deps.require(skip_steamcmd=False) + + +def test_steamcmd_is_not_required_when_it_is_skipped(monkeypatch): + monkeypatch.setattr( + deps.shutil, "which", lambda name: None if name == "steamcmd" else "/usr/bin/dd" + ) + assert deps.missing_tools(skip_steamcmd=True) == [] + assert deps.missing_tools(skip_steamcmd=False) == ["steamcmd"] + + +def test_missing_tool_raises_with_the_command_that_fixes_it(monkeypatch): + monkeypatch.setattr(deps.shutil, "which", lambda name: None) + + with pytest.raises(deps.DependencyError) as excinfo: + deps.require(skip_steamcmd=False) + + message = str(excinfo.value) + assert "depotdownloader" in message + assert "steamcmd" in message + assert "./install.sh" in message diff --git a/_installer_import/tests/test_ghost.py b/_installer_import/tests/test_ghost.py new file mode 100644 index 0000000..dbad5e0 --- /dev/null +++ b/_installer_import/tests/test_ghost.py @@ -0,0 +1,159 @@ +"""The fake-install detector. + +The anchor test uses the real manifest `steamcmd +login anonymous +app_update +284160` wrote on 2026-09-11 -- a paid app it had no licence for, reported as +"fully installed" after transferring nothing. The negatives are built from the +field signatures of the nine genuinely installed apps in the same library, +because the only failure mode that matters here is deleting a real install. + +Shared fixtures (`library`, `GHOST_ACF`, `_write`, `_real_install`) live in +conftest.py; the repair side lives in test_ghost_repair.py. +""" + +import pytest + +from steam_installer import ghost, vdf +from steam_installer.library import SteamLibrary +from tests.conftest import GHOST_ACF, _real_install, _write + + +def test_the_real_anonymous_steamcmd_manifest_is_detected(library): + _write(library, 284160, GHOST_ACF) + (library.common / "BeamNG.drive").mkdir() + + report = ghost.inspect(library, 284160) + + assert report.is_ghost + assert not report.is_warm + assert "InstalledDepots is empty" in report.reasons + assert "no game content on disk" in report.reasons + + +def test_a_missing_content_directory_is_still_a_ghost(library): + """The directory need not exist at all -- only the lie has to.""" + _write(library, 284160, GHOST_ACF) + assert ghost.inspect(library, 284160).is_ghost + + +def test_a_real_install_is_never_a_ghost(library): + _real_install(library) + report = ghost.inspect(library, 475150) + assert not report.is_ghost + assert report.content_bytes == 4096 + + +def test_content_on_disk_alone_disqualifies_a_ghost(library): + """The interlock that matters: even a manifest that looks exactly like the + fake one is not a ghost if there are real bytes behind it.""" + _write(library, 284160, GHOST_ACF) + content = library.common / "BeamNG.drive" + content.mkdir() + (content / "game.pak").write_bytes(b"x" * 1024) + + assert not ghost.inspect(library, 284160).is_ghost + + +def test_a_nonzero_size_on_disk_disqualifies_a_ghost(library): + _write( + library, 284160, GHOST_ACF.replace('"SizeOnDisk"\t\t"0"', '"SizeOnDisk"\t\t"9"') + ) + assert not ghost.inspect(library, 284160).is_ghost + + +def test_listed_depots_disqualify_a_ghost(library): + _write( + library, + 284160, + GHOST_ACF.replace( + '\t"InstalledDepots"\n\t{\n\t}', + '\t"InstalledDepots"\n\t{\n\t\t"284161"\n\t\t{\n\t\t\t"manifest"\t\t"1"\n\t\t}\n\t}', + ), + ) + assert not ghost.inspect(library, 284160).is_ghost + + +@pytest.mark.parametrize("flags", ["1026", "2", "0", "notanumber", ""]) +def test_a_manifest_not_claiming_installed_is_not_a_ghost(library, flags): + """Without bit 4 the manifest is not claiming anything is installed, so + there is no lie to clean up -- 1026 is a normal mid-download state.""" + _write( + library, + 284160, + GHOST_ACF.replace('"StateFlags"\t\t"4"', f'"StateFlags"\t\t"{flags}"'), + ) + assert not ghost.inspect(library, 284160).is_ghost + + +def test_state_flags_six_still_counts_as_claiming_installed(library): + """6 is 4|2 -- installed, update required. Every genuinely installed app in + a real library carries it, so the bit test must not be an equality test.""" + _write( + library, 284160, GHOST_ACF.replace('"StateFlags"\t\t"4"', '"StateFlags"\t\t"6"') + ) + assert ghost.inspect(library, 284160).is_ghost + + +def test_a_missing_manifest_is_not_a_ghost(library): + assert not ghost.inspect(library, 284160).is_ghost + + +def test_a_manifest_for_a_different_appid_is_ignored(library): + _write( + library, 284160, GHOST_ACF.replace('"appid"\t\t"284160"', '"appid"\t\t"999"') + ) + assert not ghost.inspect(library, 284160).is_ghost + + +def test_a_manifest_without_an_installdir_is_not_a_ghost(library): + _write(library, 284160, GHOST_ACF.replace('\t"installdir"\t\t"BeamNG.drive"\n', "")) + report = ghost.inspect(library, 284160) + assert report.install_dir is None + assert report.is_ghost + + +def test_our_own_downloader_state_disqualifies_a_ghost(library): + """An empty install dir mid-run is not a ghost; .DepotDownloader proves a + real download is in progress against it.""" + _write(library, 284160, GHOST_ACF) + cache = library.common / "BeamNG.drive" / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "284161_999.manifest").write_bytes(b"x") + + assert not ghost.inspect(library, 284160).is_ghost + + +def test_staged_content_makes_a_ghost_warm(library): + _write(library, 284160, GHOST_ACF) + staged = library.steamapps / "downloading" / "284160" + staged.mkdir(parents=True) + (staged / "part.pak").write_bytes(b"x" * 2048) + + report = ghost.inspect(library, 284160) + + assert report.is_ghost + assert report.is_warm + assert report.staging_bytes == 2048 + + +def test_scan_finds_only_the_ghost(library): + _write(library, 284160, GHOST_ACF) + _real_install(library) + (library.steamapps / "appmanifest_notanumber.acf").write_text(GHOST_ACF) + + assert [r.appid for r in ghost.scan([library])] == [284160] + + +def test_scan_skips_a_library_with_no_steamapps(tmp_path): + assert ghost.scan([SteamLibrary(tmp_path / "nope")]) == [] + + +def test_an_unparseable_manifest_is_not_a_ghost(library, monkeypatch): + """A manifest we cannot read is not a manifest we may act on.""" + _write(library, 284160, "this is not vdf {{{") + + def boom(path): + raise vdf.VdfParseError("unbalanced braces", 0) + + monkeypatch.setattr(ghost.recognize, "read_app_state", boom) + + assert not ghost.inspect(library, 284160).is_ghost diff --git a/_installer_import/tests/test_ghost_repair.py b/_installer_import/tests/test_ghost_repair.py new file mode 100644 index 0000000..34c4196 --- /dev/null +++ b/_installer_import/tests/test_ghost_repair.py @@ -0,0 +1,74 @@ +"""Clearing a ghost away, and every case where it must refuse to. + +Split from test_ghost.py to keep both files under the repo's 250-line cap. +""" + +import logging + +import pytest + +from steam_installer import ghost +from tests.conftest import GHOST_ACF, _real_install, _write + +LOGGER = logging.getLogger("test-ghost-repair") + + +def test_repair_moves_the_manifest_aside_and_removes_the_empty_directory(library): + _write(library, 284160, GHOST_ACF) + content = library.common / "BeamNG.drive" + content.mkdir() + report = ghost.inspect(library, 284160) + + touched = ghost.repair(report, LOGGER, now_unix=1757000000) + + assert not report.manifest_path.exists() + backup = library.steamapps / "appmanifest_284160.acf.ghost-1757000000" + assert backup.read_text() == GHOST_ACF + assert not content.exists() + assert touched == [backup, content] + + +def test_repair_refuses_to_delete_a_non_empty_directory(library, caplog): + """rmdir, never rmtree: the kernel itself is the last line of defence.""" + _write(library, 284160, GHOST_ACF) + content = library.common / "BeamNG.drive" + content.mkdir() + report = ghost.inspect(library, 284160) + # Content appears between the check and the repair. + (content / "surprise.pak").write_bytes(b"x") + + with caplog.at_level(logging.WARNING): + touched = ghost.repair(report, LOGGER, now_unix=1) + + assert content.exists() + assert (content / "surprise.pak").exists() + assert content not in touched + + +def test_repair_refuses_a_warm_ghost(library): + """Staged bytes are a real partial download, not a lie to clean up.""" + _write(library, 284160, GHOST_ACF) + staged = library.steamapps / "downloading" / "284160" + staged.mkdir(parents=True) + (staged / "part.pak").write_bytes(b"x" * 3000) + report = ghost.inspect(library, 284160) + + with pytest.raises(ValueError, match="partial download"): + ghost.repair(report, LOGGER) + + assert report.manifest_path.exists() + assert (staged / "part.pak").exists() + + +def test_repair_refuses_anything_that_is_not_a_ghost(library): + _real_install(library) + report = ghost.inspect(library, 475150) + + with pytest.raises(ValueError, match="not a ghost"): + ghost.repair(report, LOGGER) + + +def test_repair_defaults_its_timestamp_to_now(library): + _write(library, 284160, GHOST_ACF) + touched = ghost.repair(ghost.inspect(library, 284160), LOGGER) + assert touched[0].name.startswith("appmanifest_284160.acf.ghost-") diff --git a/_installer_import/tests/test_resolve.py b/_installer_import/tests/test_resolve.py index e9a46a3..57544d0 100644 --- a/_installer_import/tests/test_resolve.py +++ b/_installer_import/tests/test_resolve.py @@ -1,3 +1,10 @@ +"""Name -> AppID search, across both backends. + +The community endpoint is primary because the storefront one TLS-rejects +scripted clients from this network; these tests pin the fallback order, the +retry, and the exact-title rule that stops "Portal" installing Portal 2. +""" + import io import json import urllib.error @@ -6,6 +13,9 @@ import pytest from steam_installer import resolve +COMMUNITY_PAYLOAD = [{"appid": "1771300", "name": "Kingdom Come: Deliverance II"}] +STORE_PAYLOAD = {"items": [{"id": 1771300, "name": "Kingdom Come: Deliverance II"}]} + class _FakeResponse(io.BytesIO): def __enter__(self): @@ -15,12 +25,23 @@ class _FakeResponse(io.BytesIO): return False -def test_search_parses_items(monkeypatch): - payload = {"items": [{"id": 1771300, "name": "Kingdom Come: Deliverance II"}]} - monkeypatch.setattr( - resolve.urllib.request, - "urlopen", - lambda url, timeout: _FakeResponse(json.dumps(payload).encode()), +def _responder(monkeypatch, handler): + """Route a fake urlopen by URL so each backend can be driven separately.""" + seen = [] + + def fake_urlopen(request, timeout): + seen.append(request.full_url) + return handler(request.full_url) + + monkeypatch.setattr(resolve.urllib.request, "urlopen", fake_urlopen) + monkeypatch.setattr(resolve.time, "sleep", lambda _: None) + return seen + + +def test_community_backend_is_tried_first_and_parses_string_appids(monkeypatch): + """The community endpoint returns appid as a string, unlike the store one.""" + seen = _responder( + monkeypatch, lambda url: _FakeResponse(json.dumps(COMMUNITY_PAYLOAD).encode()) ) results = resolve.search("kingdom come") @@ -28,45 +49,139 @@ def test_search_parses_items(monkeypatch): assert results == [ resolve.SearchResult(appid=1771300, name="Kingdom Come: Deliverance II") ] + assert "steamcommunity.com" in seen[0] + assert len(seen) == 1 + + +def test_store_backend_is_used_when_the_community_one_fails(monkeypatch): + def handler(url): + if "steamcommunity.com" in url: + raise urllib.error.URLError("dns") + return _FakeResponse(json.dumps(STORE_PAYLOAD).encode()) + + seen = _responder(monkeypatch, handler) + + assert resolve.search("kingdom come")[0].appid == 1771300 + assert "store.steampowered.com" in seen[-1] + + +def test_a_transient_failure_is_retried_rather_than_given_up_on(monkeypatch): + """DNS here fails intermittently; one blip must not send the user off to + look up an AppID by hand.""" + attempts = [] + + def handler(url): + attempts.append(url) + if len(attempts) == 1: + raise urllib.error.URLError("temporary failure in name resolution") + return _FakeResponse(json.dumps(COMMUNITY_PAYLOAD).encode()) + + _responder(monkeypatch, handler) + + assert resolve.search("kingdom come")[0].appid == 1771300 + assert len(attempts) == 2 def test_search_quotes_the_term(monkeypatch): - seen = {} - - def fake_urlopen(url, timeout): - seen["url"] = url - return _FakeResponse(b'{"items": []}') - - monkeypatch.setattr(resolve.urllib.request, "urlopen", fake_urlopen) + seen = _responder(monkeypatch, lambda url: _FakeResponse(b"[]")) resolve.search("kingdom come/2") - assert "kingdom%20come/2" in seen["url"] or "kingdom%20come%2F2" in seen["url"] + assert "kingdom%20come/2" in seen[0] or "kingdom%20come%2F2" in seen[0] -def test_search_missing_items_key_returns_empty(monkeypatch): - monkeypatch.setattr( - resolve.urllib.request, "urlopen", lambda url, timeout: _FakeResponse(b"{}") - ) +def test_search_sends_an_explicit_user_agent(monkeypatch): + """urllib's default UA gets refused outright by Steam's edge.""" + captured = {} + + def fake_urlopen(request, timeout): + captured["ua"] = request.get_header("User-agent") + return _FakeResponse(b"[]") + + monkeypatch.setattr(resolve.urllib.request, "urlopen", fake_urlopen) + + resolve.search("anything") + + assert captured["ua"] == resolve.USER_AGENT + + +def test_rows_with_a_non_numeric_appid_are_skipped(monkeypatch): + payload = [{"appid": "nope", "name": "Broken"}, *COMMUNITY_PAYLOAD] + _responder(monkeypatch, lambda url: _FakeResponse(json.dumps(payload).encode())) + + assert [r.appid for r in resolve.search("x")] == [1771300] + + +def test_store_missing_items_key_returns_empty(monkeypatch): + def handler(url): + if "steamcommunity.com" in url: + raise urllib.error.URLError("dns") + return _FakeResponse(b"{}") + + _responder(monkeypatch, handler) assert resolve.search("nothing") == [] +def test_an_html_error_page_is_a_search_failure_not_a_crash(monkeypatch): + """A captive portal or block page returns HTML with a 200; that must not + surface as a JSONDecodeError traceback.""" + _responder(monkeypatch, lambda url: _FakeResponse(b"nope")) + + with pytest.raises(resolve.SearchUnavailable): + resolve.search("kingdom come") + + +def test_a_dict_payload_falls_through_to_the_store_parser(monkeypatch): + """The community endpoint returns a list. A dict is the store's shape, so + it is not a crash -- the store parser handles it and finds no items.""" + _responder(monkeypatch, lambda url: _FakeResponse(b'{"not": "a list"}')) + + assert resolve.search("kingdom come") == [] + + +def test_a_payload_neither_backend_understands_is_a_search_failure(monkeypatch): + _responder(monkeypatch, lambda url: _FakeResponse(b'"just a string"')) + + with pytest.raises(resolve.SearchUnavailable): + resolve.search("kingdom come") + + @pytest.mark.parametrize( "exc", [urllib.error.URLError("tls blocked"), TimeoutError("timed out")] ) def test_search_network_failure_becomes_search_unavailable(monkeypatch, exc): - """store.steampowered.com is known to reject scripted clients on some - networks -- that is not a code bug, so it gets its own exception type.""" + """Neither endpoint reachable is not a code bug, so it gets its own type.""" - def boom(url, timeout): + def boom(url): raise exc - monkeypatch.setattr(resolve.urllib.request, "urlopen", boom) + _responder(monkeypatch, boom) - with pytest.raises(resolve.SearchUnavailable): + with pytest.raises(resolve.SearchUnavailable, match="steamcommunity"): resolve.search("kingdom come") +def test_exact_match_beats_steams_own_ranking(): + """Steam ranks Portal 2 above Portal; typing the full name must still win.""" + results = [ + resolve.SearchResult(620, "Portal 2"), + resolve.SearchResult(400, "Portal"), + ] + assert resolve.exact_match(results, "portal").appid == 400 + assert resolve.exact_match(results, " PORTAL ").appid == 400 + + +def test_exact_match_returns_none_without_one(): + results = [resolve.SearchResult(620, "Portal 2")] + assert resolve.exact_match(results, "Portal") is None + + +def test_exact_match_refuses_to_pick_between_duplicate_titles(): + """Two apps with the same title is a genuine ambiguity, not a coin flip.""" + results = [resolve.SearchResult(1, "Same"), resolve.SearchResult(2, "Same")] + assert resolve.exact_match(results, "Same") is None + + def _results(): return [ resolve.SearchResult(appid=1, name="One"), diff --git a/_installer_import/tests/test_steamctl.py b/_installer_import/tests/test_steamctl.py new file mode 100644 index 0000000..0a95f14 --- /dev/null +++ b/_installer_import/tests/test_steamctl.py @@ -0,0 +1,58 @@ +"""Closing and reopening the Steam client. + +Steam is only ever asked to leave politely. A hung client is a reason to stop +and say so, never to escalate to a signal: SIGKILL mid-write is how a manifest +ends up truncated. +""" + +import logging + +import pytest + +from steam_installer import recognize, steamctl + +LOGGER = logging.getLogger("test-steamctl") + + +def test_stop_is_a_no_op_when_steam_is_not_running(monkeypatch): + monkeypatch.setattr(recognize, "steam_client_pids", list) + called = [] + monkeypatch.setattr(steamctl.subprocess, "run", lambda *a, **k: called.append(a)) + + assert steamctl.stop(LOGGER) is False + assert called == [] + + +def test_stop_asks_steam_to_shut_down_and_waits_for_it(monkeypatch): + pids = iter([[42], [42], [], []]) + monkeypatch.setattr(recognize, "steam_client_pids", lambda: next(pids)) + seen = {} + monkeypatch.setattr( + steamctl.subprocess, "run", lambda cmd, **k: seen.setdefault("cmd", cmd) + ) + + assert steamctl.stop(LOGGER, sleep=lambda _: None) is True + assert seen["cmd"] == ["steam", "-shutdown"] + + +def test_stop_gives_up_rather_than_killing_a_client_that_will_not_exit(monkeypatch): + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [42]) + monkeypatch.setattr(steamctl.subprocess, "run", lambda *a, **k: None) + + with pytest.raises(steamctl.SteamStillRunning, match="42"): + steamctl.stop(LOGGER, timeout=1.0, sleep=lambda _: None) + + +def test_start_launches_steam_detached(monkeypatch): + seen = {} + + def fake_popen(cmd, **kwargs): + seen["cmd"] = cmd + seen["kwargs"] = kwargs + + monkeypatch.setattr(steamctl.subprocess, "Popen", fake_popen) + + steamctl.start(LOGGER) + + assert seen["cmd"] == ["steam"] + assert seen["kwargs"]["start_new_session"] is True diff --git a/_installer_import/tests/test_takeover.py b/_installer_import/tests/test_takeover.py new file mode 100644 index 0000000..de3777b --- /dev/null +++ b/_installer_import/tests/test_takeover.py @@ -0,0 +1,134 @@ +"""Adopting content the Steam client already downloaded. + +The bytes in `steamapps/downloading//` are a real partial download the +user waited for. A tool whose purpose is speed must reuse them, and must not +destroy them before the install it feeds has actually succeeded. +""" + +import logging + +import pytest + +from steam_installer import takeover +from steam_installer.library import SteamLibrary + +LOGGER = logging.getLogger("test-takeover") + + +@pytest.fixture +def library(tmp_path): + lib = SteamLibrary(tmp_path / "Steam") + lib.steamapps.mkdir(parents=True) + lib.common.mkdir(parents=True) + return lib + + +def _stage(library, appid, files): + staged = library.steamapps / "downloading" / str(appid) + for name, payload in files.items(): + path = staged / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + return staged + + +def test_no_staging_directory_means_nothing_to_adopt(library): + assert takeover.find_staged(library, 284160, library.common / "Game") is None + + +def test_an_empty_staging_directory_means_nothing_to_adopt(library): + (library.steamapps / "downloading" / "284160").mkdir(parents=True) + assert takeover.find_staged(library, 284160, library.common / "Game") is None + + +def test_find_staged_measures_the_whole_tree(library): + _stage(library, 284160, {"Bin64/game.exe": b"x" * 100, "content/a.zip": b"y" * 50}) + + staged = takeover.find_staged(library, 284160, library.common / "Game") + + assert staged.file_count == 2 + assert staged.bytes_staged == 150 + + +def test_adoption_hardlinks_rather_than_moving_or_copying(library): + """Nothing is destroyed before the install is proven, and a 47 GB tree costs + no extra disk.""" + source = _stage(library, 284160, {"Bin64/game.exe": b"x" * 100}) + install_dir = library.common / "Game" + install_dir.mkdir() + staged = takeover.find_staged(library, 284160, install_dir) + + result = takeover.adopt(staged, LOGGER) + + adopted = install_dir / "Bin64" / "game.exe" + assert adopted.read_bytes() == b"x" * 100 + assert (source / "Bin64" / "game.exe").exists() + # One inode, two names -- so it cost no extra disk. + assert adopted.stat().st_ino == (source / "Bin64" / "game.exe").stat().st_ino + assert result.linked == 1 + assert result.bytes_adopted == 100 + assert result.failed == 0 + + +def test_a_patched_file_is_corrected_in_both_trees(library): + """depotdownloader seeks and writes into an existing file rather than + replacing it. Through a shared inode that also repairs Steam's staged copy, + so whichever downloader finishes, the other one's view is already right.""" + source = _stage(library, 284160, {"part.pak": b"old-content"}) + install_dir = library.common / "Game" + install_dir.mkdir() + takeover.adopt(takeover.find_staged(library, 284160, install_dir), LOGGER) + + # Stand in for depotdownloader: open the adopted file and patch it in place. + with open(install_dir / "part.pak", "r+b") as handle: + handle.seek(0) + handle.write(b"new") + + assert (source / "part.pak").read_bytes() == b"new-content" + + +def test_adoption_never_overwrites_a_file_already_in_place(library): + _stage(library, 284160, {"game.pak": b"staged"}) + install_dir = library.common / "Game" + install_dir.mkdir() + (install_dir / "game.pak").write_bytes(b"already here") + staged = takeover.find_staged(library, 284160, install_dir) + + result = takeover.adopt(staged, LOGGER) + + assert (install_dir / "game.pak").read_bytes() == b"already here" + assert result.skipped == 1 + assert result.linked == 0 + + +def test_a_link_failure_is_survivable(library, monkeypatch, caplog): + """depotdownloader simply fetches whatever could not be adopted, so a failed + link is slower, never wrong.""" + _stage(library, 284160, {"a.pak": b"x", "b.pak": b"y"}) + install_dir = library.common / "Game" + install_dir.mkdir() + staged = takeover.find_staged(library, 284160, install_dir) + + def refuse(src, dst): + raise OSError("cross-device link") + + monkeypatch.setattr(takeover.os, "link", refuse) + + with caplog.at_level(logging.WARNING): + result = takeover.adopt(staged, LOGGER) + + assert result.failed == 2 + assert result.linked == 0 + + +def test_symlinks_in_the_staging_tree_are_ignored(library): + staged_dir = _stage(library, 284160, {"real.pak": b"x" * 10}) + (staged_dir / "link.pak").symlink_to(staged_dir / "real.pak") + install_dir = library.common / "Game" + install_dir.mkdir() + + staged = takeover.find_staged(library, 284160, install_dir) + result = takeover.adopt(staged, LOGGER) + + assert staged.file_count == 1 + assert result.linked == 1 diff --git a/_installer_import/tests/test_target.py b/_installer_import/tests/test_target.py new file mode 100644 index 0000000..f02495f --- /dev/null +++ b/_installer_import/tests/test_target.py @@ -0,0 +1,178 @@ +"""Turning one free-form argument into a confirmed Steam app. + +Four shapes have to work -- an AppID, a Steam URL, an exact title and a +partial title -- and the ambiguous cases have to fail rather than guess. +""" + +import pytest + +from steam_installer import appinfo, resolve, target + + +def _metadata(appid): + return appinfo.AppMetadata(appid, "BeamNG.drive", "BeamNG.drive") + + +@pytest.fixture +def steam_knows_the_app(monkeypatch): + monkeypatch.setattr(appinfo, "get_metadata", _metadata) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("284160", (284160, "appid")), + (" 284160 ", (284160, "appid")), + ("https://steamcommunity.com/app/284160/guides/", (284160, "url")), + ("https://store.steampowered.com/app/284160/BeamNGdrive/", (284160, "url")), + ("https://store.steampowered.com/app/284160", (284160, "url")), + ("https://store.steampowered.com/app/284160/?snr=1_7", (284160, "url")), + ("steamcommunity.com/app/284160/", (284160, "url")), + ("BeamNG.drive", None), + ("BeamNG", None), + ("0", None), + ("https://steamcommunity.com/id/someone/", None), + ], +) +def test_parse_literal_recognises_appids_and_urls(text, expected): + assert target.parse_literal(text) == expected + + +def test_an_appid_is_confirmed_through_steamcmd(steam_knows_the_app): + resolved = target.resolve_target("284160", None, interactive=False) + assert resolved.metadata.appid == 284160 + assert resolved.source == "appid" + + +def test_the_appid_flag_skips_detection_entirely(steam_knows_the_app): + resolved = target.resolve_target("ignored name", 284160, interactive=False) + assert resolved.metadata.appid == 284160 + assert resolved.source == "flag" + + +def test_a_url_resolves_without_searching(monkeypatch, steam_knows_the_app): + def no_search(name): + raise AssertionError("a URL must not trigger a search") + + monkeypatch.setattr(resolve, "search", no_search) + resolved = target.resolve_target( + "https://steamcommunity.com/app/284160/guides/", None, interactive=False + ) + assert resolved.source == "url" + + +def test_a_single_search_hit_is_taken(monkeypatch, steam_knows_the_app): + monkeypatch.setattr( + resolve, "search", lambda name: [resolve.SearchResult(284160, "BeamNG.drive")] + ) + resolved = target.resolve_target("BeamNG", None, interactive=False) + assert resolved.metadata.appid == 284160 + assert resolved.source == "only match" + + +def test_an_exact_title_wins_over_a_higher_ranked_sequel(monkeypatch): + """Steam ranks Portal 2 first for "Portal"; typing the full name must not + install the sequel.""" + monkeypatch.setattr( + resolve, + "search", + lambda name: [ + resolve.SearchResult(620, "Portal 2"), + resolve.SearchResult(400, "Portal"), + ], + ) + monkeypatch.setattr( + appinfo, "get_metadata", lambda appid: appinfo.AppMetadata(appid, "Portal", "P") + ) + + resolved = target.resolve_target("Portal", None, interactive=False) + + assert resolved.metadata.appid == 400 + assert resolved.source == "exact" + + +def test_an_ambiguous_name_prompts_when_interactive(monkeypatch, steam_knows_the_app): + results = [resolve.SearchResult(1, "Witcher 3"), resolve.SearchResult(2, "Witcher")] + monkeypatch.setattr(resolve, "search", lambda name: results) + monkeypatch.setattr(resolve, "prompt_confirm", lambda r: r[1]) + + resolved = target.resolve_target("witch", None, interactive=True) + + assert resolved.source == "picked" + + +def test_an_ambiguous_name_fails_closed_when_not_interactive( + monkeypatch, steam_knows_the_app, capsys +): + """Guessing here is how the wrong 90 GB gets downloaded.""" + monkeypatch.setattr( + resolve, + "search", + lambda name: [ + resolve.SearchResult(1, "Witcher 3"), + resolve.SearchResult(2, "Witcher 2"), + ], + ) + + with pytest.raises(target.TargetError, match="Ambiguous"): + target.resolve_target("witch", None, interactive=False) + + err = capsys.readouterr().err + assert "Witcher 3" in err + assert "Witcher 2" in err + + +def test_no_matches_is_an_error_not_a_default_pick(monkeypatch): + monkeypatch.setattr(resolve, "search", lambda name: []) + with pytest.raises(target.TargetError, match="No Steam app matches"): + target.resolve_target("kwyjibo", None, interactive=False) + + +def test_search_being_unavailable_points_at_the_appid_workaround(monkeypatch): + def boom(name): + raise resolve.SearchUnavailable("tls blocked") + + monkeypatch.setattr(resolve, "search", boom) + + with pytest.raises(target.TargetError, match="numeric AppID"): + target.resolve_target("BeamNG", None, interactive=False) + + +def test_digits_that_are_not_an_appid_fall_through_to_a_name_search(monkeypatch): + """A game literally called "2048" exists, so digits are not unambiguous.""" + tried = [] + + def metadata(appid): + tried.append(appid) + if appid == 2048: + raise appinfo.AppInfoError("no such app") + return appinfo.AppMetadata(appid, "2048", "2048") + + monkeypatch.setattr(appinfo, "get_metadata", metadata) + monkeypatch.setattr( + resolve, "search", lambda name: [resolve.SearchResult(999, "2048")] + ) + + resolved = target.resolve_target("2048", None, interactive=False) + + assert resolved.metadata.appid == 999 + assert tried == [2048, 999] + + +def test_an_unknown_appid_from_a_url_does_not_fall_through(monkeypatch): + """A URL cannot also be a game's name, so there is nothing to fall back to.""" + + def boom(appid): + raise appinfo.AppInfoError("no such app") + + monkeypatch.setattr(appinfo, "get_metadata", boom) + + with pytest.raises(target.TargetError, match="not a known Steam app"): + target.resolve_target( + "https://store.steampowered.com/app/1/x/", None, interactive=False + ) + + +def test_no_target_at_all_is_an_error(): + with pytest.raises(target.TargetError, match="No target given"): + target.resolve_target(None, None, interactive=False) From 8f641e235d0721a86ddfeeb21bddfc87f04e1cf6 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 25 Sep 2026 18:41:55 +0200 Subject: [PATCH 11/24] Fetch owned DLC with depotdownloader too; split recognize.py; enforce the 250-line cap Co-Authored-By: Claude Opus 5 (1M context) --- .../.github/workflows/file-length.yml | 35 ++ _installer_import/.pre-commit-config.yaml | 9 + _installer_import/README.md | 24 + _installer_import/TODO-file-length-250.md | 86 --- .../scripts/check_file_length.sh | 31 + _installer_import/steam_installer/appinfo.py | 100 +++- _installer_import/steam_installer/depotdl.py | 71 ++- _installer_import/steam_installer/deps.py | 21 +- _installer_import/steam_installer/dlc.py | 60 ++ _installer_import/steam_installer/finalize.py | 7 +- .../steam_installer/finish_check.py | 97 +++ _installer_import/steam_installer/flow.py | 21 +- _installer_import/steam_installer/ghost.py | 4 +- .../steam_installer/manifest_write.py | 191 ++++++ .../steam_installer/recognize.py | 335 +---------- .../steam_installer/steamcmd_validate.py | 79 +++ _installer_import/tests/_cli_helpers.py | 21 + _installer_import/tests/_recognize_helpers.py | 31 + _installer_import/tests/conftest.py | 65 +- _installer_import/tests/test_cli.py | 207 +------ _installer_import/tests/test_cli_finish.py | 118 ++++ _installer_import/tests/test_dlc.py | 184 ++++++ _installer_import/tests/test_finish_check.py | 159 +++++ .../tests/test_manifest_finish.py | 187 ++++++ .../tests/test_manifest_write.py | 157 +++++ _installer_import/tests/test_recognize.py | 565 +----------------- .../tests/test_steamcmd_validate.py | 47 ++ 27 files changed, 1705 insertions(+), 1207 deletions(-) create mode 100644 _installer_import/.github/workflows/file-length.yml delete mode 100644 _installer_import/TODO-file-length-250.md create mode 100755 _installer_import/scripts/check_file_length.sh create mode 100644 _installer_import/steam_installer/dlc.py create mode 100644 _installer_import/steam_installer/finish_check.py create mode 100644 _installer_import/steam_installer/manifest_write.py create mode 100644 _installer_import/steam_installer/steamcmd_validate.py create mode 100644 _installer_import/tests/_cli_helpers.py create mode 100644 _installer_import/tests/_recognize_helpers.py create mode 100644 _installer_import/tests/test_cli_finish.py create mode 100644 _installer_import/tests/test_dlc.py create mode 100644 _installer_import/tests/test_finish_check.py create mode 100644 _installer_import/tests/test_manifest_finish.py create mode 100644 _installer_import/tests/test_manifest_write.py create mode 100644 _installer_import/tests/test_steamcmd_validate.py diff --git a/_installer_import/.github/workflows/file-length.yml b/_installer_import/.github/workflows/file-length.yml new file mode 100644 index 0000000..38f5df1 --- /dev/null +++ b/_installer_import/.github/workflows/file-length.yml @@ -0,0 +1,35 @@ +name: file length + +# The shared 250-line cap, enforced on push as well as at commit time: the +# pre-commit hook only sees staged files and a fresh clone may never have run +# `pre-commit install`. This checks the whole tree. + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + file-length: + name: files at most 250 lines + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # The gate and its exemption list live in the shared utils repo. + - name: Check out the shared gate + uses: actions/checkout@v4 + with: + repository: kuhyx/utils + path: .utils + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + # Tracked files only: --all would also walk the .utils checkout above, + # which is another repo's code and not this one's to split. + - name: Every file is at most 250 lines + run: git ls-files -z | xargs -0 bash .utils/scripts/check_file_length.sh diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml index e76039b..ef2f130 100644 --- a/_installer_import/.pre-commit-config.yaml +++ b/_installer_import/.pre-commit-config.yaml @@ -16,6 +16,15 @@ repos: language: system files: \.(md|markdown)$ + # Shared 250-line cap on every file in the commit, code and prose alike. + # Checks only what is staged, so an unrelated commit never goes red, and + # it fails rather than warns: no baseline, no allowlist. + - id: file-length + name: files at most 250 lines + entry: scripts/check_file_length.sh + language: system + types: [text] + # Every dependency on its ecosystem's newest STABLE release, exact-pinned # so the question is mechanically answerable at all. # diff --git a/_installer_import/README.md b/_installer_import/README.md index 08c1823..c8eeab9 100644 --- a/_installer_import/README.md +++ b/_installer_import/README.md @@ -51,6 +51,29 @@ place also rewrites Steam's copy. That is deliberate — both names converge on the same corrected content — but it does mean the staged bytes are working data, not a frozen backup. On ext4 there is no reflink that would give you both. +## Owned DLC comes down fast too + +Some DLC keeps its content under an app id of its own rather than in the +game's depot list -- Cyberpunk 2077's Phantom Liberty is app 2138330, about +25 GB. A depotdownloader run for the game never sees it, and Steam, restarted +onto a manifest without it, downloads all of it at its own pace. + +So every DLC in the game's `listofdlc` that has depots of its own gets its own +depotdownloader run into the same install dir. Depotdownloader's licence check +decides ownership: its "not available from this account" is skipped as +expected, any other failure aborts before the manifest is touched. Each DLC +depot is then written into `InstalledDepots` with its `dlcappid`, exactly as +Steam records it. + +## Which depotdownloader + +Always `~/.local/bin/depotdownloader` (the copy `install.sh` manages) when it +exists, never "whatever is first on PATH". Each depotdownloader binary keeps +its remembered login in its own storage slot, keyed by its path; with an AUR +copy in `/usr/bin` too, a terminal run and a `sudo`'d run (whose PATH lacks +`~/.local/bin`) used different binaries -- one logged in, one prompting for a +password nobody was there to type. + ## Ghost installs `steamcmd +login anonymous +app_update ` has no licence for the app. @@ -95,6 +118,7 @@ terminal — with a plain pipe the prompt never appears and the run looks hung. ## How it works 1. `depotdownloader` pulls the content (fast path). + Then each owned DLC with depots of its own gets a run of its own. 2. `steamcmd app_update validate` reconciles Steam's own bookkeeping. 3. If that fails or leaves the manifest incomplete, the manifest is finished by hand — but only when every precondition below holds. diff --git a/_installer_import/TODO-file-length-250.md b/_installer_import/TODO-file-length-250.md deleted file mode 100644 index 9f33c98..0000000 --- a/_installer_import/TODO-file-length-250.md +++ /dev/null @@ -1,86 +0,0 @@ -# Refactor TODO — enforce the 250-line file cap - -> **This file is a ready-to-use prompt.** Paste it to Claude, or open this repo -> and say "do refactor_claude_todo". It is self-contained: everything needed to -> execute is below. Generated 2026-08-14 from a measured survey of every repo. - -## Goal - -Every file in this repo must be **at most 250 lines** — source, tests, and -prose (`.md`/`.txt`/`.rst`/`.tex`) alike — and must **stay** that way forever, -enforced by a gate that fails the commit, not by a note anyone can ignore. - -Why: a file that cannot be read in one piece forces re-reads and partial edits, -which is the single largest avoidable cost in an LLM-assisted workflow. Aim by -churn, not size alone — refactoring pays where code is read and changed often -(Fowler, _refactoring economic benefit_). - -## Scope in this repo - -- **4 files** currently exceed 250 lines (of 21 eligible files). -- **1,877 lines** sit in violation; longest file is **765 lines**. - -Exempt (do NOT split these): - -- generated files — `*.g.dart`, `*.freezed.dart`, `*.gr.dart`, `**/l10n/generated/**`, - anything with a `GENERATED` header -- markup — `.html`, `.css`, `.scss` -- data files — `.json`, `.yaml`, `.csv`, wordlists and other data-ish `.txt` - (mean line length under 25 chars) - -## Violations, highest ROI first - -ROI = lines x commits in the last year. Work top-down; a long file nobody edits -has near-zero payoff and should not be first. - -| lines | commits/yr | kind | file | -| ----: | ---------: | :--- | :----------------------------- | -| 765 | 3 | code | `tests/test_recognize.py` | -| 473 | 3 | code | `steam_installer/recognize.py` | -| 311 | 3 | code | `steam_installer/cli.py` | -| 328 | 1 | code | `tests/test_cli.py` | - -## How to split - -- **Python** — extract cohesive helpers into sibling modules; keep the public - API and imports stable. -- **Shell** — split into `lib/*.sh` sourced by a thin entry script. Keep - `set -euo pipefail` in each. -- **Dart / TypeScript** — extract widgets/components into their own files. -- **Tests** — split by test-group into sibling files - (`foo.test.ts` -> `foo.parsing.test.ts`, `foo.render.test.ts`). Coverage must - not drop. -- **Docs** — split into topic files under `docs/` with an index. For an - oversized `CLAUDE.md`, move detail into referenced docs so the - always-loaded part shrinks. - -**Do not** game the cap: no one-lining, no deleting tests, no moving code into -an exempt extension, no `# noqa`-style suppressions. - -## Make it permanent (required — this is the point) - -A refactor without a gate silently regrows. Before this task is done: - -1. Wire the shared gate `~/src/utils/scripts/check_file_length.sh` into this repo's - `.pre-commit-config.yaml` as a local hook. If the repo has no pre-commit - config, add a minimal one. -2. The hook checks **files in the commit** (not the whole tree), so unrelated - commits never break, and it **fails** — exit 1, not a warning. -3. No baseline file and no allowlist. Those are suppressions. -4. If this repo has CI (`.github/workflows`), add the same check there so it - also fails on push. - -## Done condition - -- `bash ~/src/utils/scripts/check_file_length.sh --all` from this repo root exits 0. -- The repo's own test suite and coverage bar are still green. -- `pre-commit run --files ` passes. -- A deliberately over-250-line test file, staged, makes `git commit` **fail**. -- For a deployed daemon/app: the entry point still actually runs. - -## Verify - -Run the repo's own suite, then run the program itself and confirm the -behaviour is unchanged. Testing is the last step, never the first. - -REMOVE ME AFTER FINISH diff --git a/_installer_import/scripts/check_file_length.sh b/_installer_import/scripts/check_file_length.sh new file mode 100755 index 0000000..915b47c --- /dev/null +++ b/_installer_import/scripts/check_file_length.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# ============================================================================ +# Fail if any file in the commit exceeds the shared 250-line cap. +# +# Thin delegate to the shared gate in ~/src/utils, which owns the cap and the +# exemption list (generated files, markup, data). Copying that logic here is +# how one repo's idea of "too long" drifts from every other repo's -- so this +# script only locates the shared checker and forwards its arguments. +# +# Usage: +# scripts/check_file_length.sh [ ...] # pre-commit passes these +# scripts/check_file_length.sh --all # whole tree, from cwd +# ============================================================================ + +set -euo pipefail + +readonly SHARED_GATE="${UTILS_ROOT:-$HOME/src/utils}/scripts/check_file_length.sh" + +main() { + if [[ ! -x "$SHARED_GATE" ]]; then + echo "Error: shared file-length gate not found at $SHARED_GATE" >&2 + echo " Clone github.com/kuhyx/utils to ~/src/utils, or set" >&2 + echo " UTILS_ROOT to where it lives." >&2 + exit 1 + fi + + exec bash "$SHARED_GATE" "$@" +} + +main "$@" diff --git a/_installer_import/steam_installer/appinfo.py b/_installer_import/steam_installer/appinfo.py index a238382..a32c654 100644 --- a/_installer_import/steam_installer/appinfo.py +++ b/_installer_import/steam_installer/appinfo.py @@ -4,7 +4,7 @@ from __future__ import annotations import re import subprocess -from dataclasses import dataclass +from dataclasses import dataclass, field from steam_installer import vdf @@ -31,13 +31,22 @@ class DepotPlan: required: set[str] shared: set[str] + # depot id -> the DLC app it belongs to. Steam writes that dlcappid into + # InstalledDepots, and a DLC depot recorded without it is not credited. + dlc: dict[str, str] = field(default_factory=dict) @property def all_expected(self) -> set[str]: return self.required | self.shared -def _fetch_app_block(appid: int) -> dict: +def _fetch_app_blocks(appids: list[int]) -> dict[int, dict]: + """App info for every id in `appids`, from a single anonymous steamcmd call. + + One call for the lot: each steamcmd start costs seconds of login, and a game + can list dozens of DLC. + """ + prints = [arg for appid in appids for arg in ("+app_info_print", str(appid))] result = subprocess.run( [ "steamcmd", @@ -47,25 +56,39 @@ def _fetch_app_block(appid: int) -> dict: "windows", "+login", "anonymous", - "+app_info_print", - str(appid), + *prints, "+quit", ], capture_output=True, text=True, - timeout=60, + # The exit code says nothing useful; a missing block is caught below. + check=False, + timeout=60 + 5 * len(appids), ) output = result.stdout - # app_info_print emits `""\n{ ... }` inside a stream of steamcmd's own log lines; - # slice from the first line that starts the VDF block onward. - match = re.search(rf'^"{appid}"\s*$', output, re.MULTILINE) - if not match: - raise AppInfoError(f"steamcmd did not return app info for {appid}:\n{output}") - data = vdf.parse(output[match.start() :]) - app_block = data.get(str(appid), {}) - if not isinstance(app_block, dict): - raise AppInfoError(f"unexpected app_info_print structure for {appid}") - return app_block + # app_info_print emits `""\n{ ... }` inside a stream of steamcmd's own + # log lines; each block runs from its own header to the next one. + starts = {} + for appid in appids: + match = re.search(rf'^"{appid}"\s*$', output, re.MULTILINE) + if not match: + raise AppInfoError( + f"steamcmd did not return app info for {appid}:\n{output}" + ) + starts[appid] = match.start() + ordered = sorted(starts.values()) + [len(output)] + blocks: dict[int, dict] = {} + for appid, start in starts.items(): + end = ordered[ordered.index(start) + 1] + app_block = vdf.parse(output[start:end]).get(str(appid), {}) + if not isinstance(app_block, dict): + raise AppInfoError(f"unexpected app_info_print structure for {appid}") + blocks[appid] = app_block + return blocks + + +def _fetch_app_block(appid: int) -> dict: + return _fetch_app_blocks([appid])[appid] def get_metadata(appid: int) -> AppMetadata: @@ -118,7 +141,49 @@ def get_depot_plan( depots = app_block.get("depots", {}) if not isinstance(depots, dict): raise AppInfoError(f"no 'depots' section in app info for {appid}") + plan = _plan_from_depots(appid, depots, language, os_name) + # Tag the game's own list's DLC depots too: depotdownloader fetches the + # owned ones in the base run, and Steam records each with its dlcappid. + for depot_id, entry in depots.items(): + if isinstance(entry, dict) and isinstance(entry.get("dlcappid"), str): + plan.dlc[depot_id] = entry["dlcappid"] + return plan + +def get_dlc_plans( + appid: int, language: str = "english", os_name: str = "windows" +) -> dict[int, set[str]]: + """DLC app id -> the depots it installs itself, for DLC with depots of its own. + + Some DLC (Cyberpunk 2077's Phantom Liberty, app 2138330, ~25 GB) keeps its + content under its own app id rather than in the game's depot list, so a + depotdownloader run for the game never sees it. DLC with no depots of its + own, or whose depots the game already lists, needs no run of its own. + """ + app_block = _fetch_app_block(appid) + extended = app_block.get("extended", {}) + listed = extended.get("listofdlc", "") if isinstance(extended, dict) else "" + dlc_ids = [int(x) for x in str(listed).split(",") if x.strip().isdigit()] + if not dlc_ids: + return {} + game_depots = app_block.get("depots", {}) + game_depot_ids = set(game_depots) if isinstance(game_depots, dict) else set() + plans: dict[int, set[str]] = {} + for dlc_id, dlc_block in _fetch_app_blocks(dlc_ids).items(): + depots = dlc_block.get("depots", {}) + if not isinstance(depots, dict): + continue + own = _plan_from_depots(dlc_id, depots, language, os_name).required + own -= game_depot_ids + if own: + plans[dlc_id] = own + return plans + + +def _plan_from_depots( + appid: int, depots: dict, language: str, os_name: str +) -> DepotPlan: + """The depots of one app's `depots` section that an (os, language) install gets.""" required: set[str] = set() shared: set[str] = set() for depot_id, entry in depots.items(): @@ -128,7 +193,10 @@ def get_depot_plan( if "depotfromapp" in entry or "sharedinstall" in entry: shared.add(depot_id) continue - if entry.get("dlcappid") or entry.get("optional") == "1": + # Another app's DLC is not part of this install -- but a DLC app's own + # depots carry its own id as dlcappid, and are exactly what it installs. + dlc_owner = entry.get("dlcappid") + if (dlc_owner and dlc_owner != str(appid)) or entry.get("optional") == "1": continue config = entry.get("config", {}) config = config if isinstance(config, dict) else {} diff --git a/_installer_import/steam_installer/depotdl.py b/_installer_import/steam_installer/depotdl.py index 09b5cf3..9349bf0 100644 --- a/_installer_import/steam_installer/depotdl.py +++ b/_installer_import/steam_installer/depotdl.py @@ -14,11 +14,42 @@ from __future__ import annotations import logging from pathlib import Path -from steam_installer import runlog +from steam_installer import deps, runlog DEFAULT_MAX_DOWNLOADS = 32 +# depotdownloader's licence check refusing a DLC app. The one failure that is +# expected: most games list DLC the account never bought. +NOT_OWNED_MARKER = "is not available from this account" + + +def _command( + appid: int, + install_dir: Path, + username: str, + language: str, + os_name: str, + max_downloads: int, +) -> list[str]: + return [ + deps.depotdownloader_bin(), + "-app", + str(appid), + "-os", + os_name, + "-language", + language, + "-max-downloads", + str(max_downloads), + "-username", + username, + "-remember-password", + "-dir", + str(install_dir), + ] + + def download( appid: int, install_dir: Path, @@ -35,21 +66,27 @@ def download( whether touching the manifest is safe at all. """ install_dir.mkdir(parents=True, exist_ok=True) - cmd = [ - "depotdownloader", - "-app", - str(appid), - "-os", - os_name, - "-language", - language, - "-max-downloads", - str(max_downloads), - "-username", - username, - "-remember-password", - "-dir", - str(install_dir), - ] + cmd = _command(appid, install_dir, username, language, os_name, max_downloads) returncode, _ = runlog.run_logged(cmd, logger) return returncode + + +def download_dlc( + dlc_appid: int, + install_dir: Path, + username: str, + logger: logging.Logger, + language: str = "english", + os_name: str = "windows", + max_downloads: int = DEFAULT_MAX_DOWNLOADS, +) -> bool | None: + """Fetch one DLC app into the game's own `install_dir`. + + Returns True when downloaded, None when the account does not own it, and + False for any other failure -- which the caller must treat as fatal. + """ + cmd = _command(dlc_appid, install_dir, username, language, os_name, max_downloads) + returncode, output = runlog.run_logged(cmd, logger) + if returncode == 0: + return True + return None if NOT_OWNED_MARKER in output else False diff --git a/_installer_import/steam_installer/deps.py b/_installer_import/steam_installer/deps.py index 38e8f82..ee95e39 100644 --- a/_installer_import/steam_installer/deps.py +++ b/_installer_import/steam_installer/deps.py @@ -2,7 +2,9 @@ from __future__ import annotations +import os import shutil +from pathlib import Path TOOL_HINTS = { "depotdownloader": "Run ./install.sh, which fetches it from the SteamRE release.", @@ -10,16 +12,33 @@ TOOL_HINTS = { } +# The copy install.sh manages. Preferred over whatever PATH finds first: each +# depotdownloader binary keeps its remembered login in its own IsolatedStorage +# slot, keyed by its path. With an AUR copy in /usr/bin as well, a terminal run +# (PATH has ~/.local/bin) and a sudo'd run (secure_path does not) used two +# different binaries -- one logged in, one asking for a password nobody was +# there to type. +MANAGED_DEPOTDOWNLOADER = Path.home() / ".local" / "bin" / "depotdownloader" + + class DependencyError(Exception): pass +def depotdownloader_bin() -> str: + """The depotdownloader to run: install.sh's own copy, else PATH's.""" + if os.access(MANAGED_DEPOTDOWNLOADER, os.X_OK): + return str(MANAGED_DEPOTDOWNLOADER) + return "depotdownloader" + + def missing_tools(*, skip_steamcmd: bool) -> list[str]: """Which required binaries are not on PATH, in the order they'd be used.""" needed = ["depotdownloader"] if not skip_steamcmd: needed.append("steamcmd") - return [tool for tool in needed if shutil.which(tool) is None] + resolved = {"depotdownloader": depotdownloader_bin()} + return [tool for tool in needed if shutil.which(resolved.get(tool, tool)) is None] def require(*, skip_steamcmd: bool) -> None: diff --git a/_installer_import/steam_installer/dlc.py b/_installer_import/steam_installer/dlc.py new file mode 100644 index 0000000..11f2c20 --- /dev/null +++ b/_installer_import/steam_installer/dlc.py @@ -0,0 +1,60 @@ +"""Owned DLC that ships as an app of its own, fetched next to the base game. + +depotdownloader's `-app ` run only walks the game's own depot list. DLC +such as Cyberpunk 2077's Phantom Liberty (app 2138330, ~25 GB) keeps its depot +under its own app id, so that run never touches it -- and the Steam client, +restarted onto a manifest without it, then downloaded all 24 GB at its own +pace. Each such DLC now gets a depotdownloader run of its own into the same +install dir, where the manifest step picks its depot up from the cache. + +Ownership is left to depotdownloader's licence check. Its refusal is the one +failure that is expected (games list DLC nobody bought); anything else aborts +the install before the manifest is touched, like a failed base download. +""" + +from __future__ import annotations + +from pathlib import Path + +from steam_installer import appinfo, depotdl, runlog + + +def download_owned( + appid: int, + install_dir: Path, + username: str, + language: str, + max_downloads: int, + log: runlog.RunLog, +) -> dict[str, str] | None: + """Download every owned DLC of `appid` that has depots of its own. + + Returns: + depot id -> DLC app id for each DLC depot now downloaded, or None if a + DLC download failed for a reason other than not owning it. + """ + logger = log.logger + plans = appinfo.get_dlc_plans(appid, language) + downloaded: dict[str, str] = {} + for dlc_appid, depots in sorted(plans.items()): + log.section(f"depotdownloader DLC {dlc_appid}") + outcome = depotdl.download_dlc( + dlc_appid=dlc_appid, + install_dir=install_dir, + username=username, + language=language, + max_downloads=max_downloads, + logger=logger, + ) + if outcome is None: + logger.info("DLC %d is not owned by this account -- skipped.", dlc_appid) + continue + if not outcome: + logger.error( + "depotdownloader failed for DLC %d -- not touching the manifest.", + dlc_appid, + ) + return None + logger.info("Downloaded DLC %d (depots %s).", dlc_appid, sorted(depots)) + downloaded.update(dict.fromkeys(depots, str(dlc_appid))) + return downloaded diff --git a/_installer_import/steam_installer/finalize.py b/_installer_import/steam_installer/finalize.py index dd95413..cfff1d1 100644 --- a/_installer_import/steam_installer/finalize.py +++ b/_installer_import/steam_installer/finalize.py @@ -10,7 +10,7 @@ from __future__ import annotations import time from pathlib import Path -from steam_installer import appinfo, recognize, runlog +from steam_installer import appinfo, finish_check, manifest_write, recognize, runlog from steam_installer.library import SteamLibrary @@ -47,7 +47,7 @@ def finish( return 1 app_state = recognize.read_app_state(manifest_path) - problems = recognize.check_ready_to_finish( + problems = finish_check.check_ready_to_finish( install_dir=target, app_state=app_state, required_depots=depot_plan.required, @@ -70,11 +70,12 @@ def finish( if freed: logger.info("Cleared %.1f GB of now-redundant Steam staging data.", freed / 1e9) - written = recognize.finish_manifest_manually( + written = manifest_write.finish_manifest_manually( library=library, appid=metadata.appid, install_dir=target, now_unix=int(time.time()), + dlc_depots=depot_plan.dlc, ) log.file_contents("Manifest after manual finish", written) diff --git a/_installer_import/steam_installer/finish_check.py b/_installer_import/steam_installer/finish_check.py new file mode 100644 index 0000000..50d5694 --- /dev/null +++ b/_installer_import/steam_installer/finish_check.py @@ -0,0 +1,97 @@ +"""The fail-closed preconditions for hand-finishing a manifest. + +Split out of recognize.py (250-line cap). Everything here only *reads*; the +write it gates lives in :mod:`manifest_write`. +""" + +from __future__ import annotations + +from pathlib import Path + +from steam_installer import manifest_write, recognize + + +def check_ready_to_finish( + install_dir: Path, + app_state: dict, + required_depots: set[str], + expected_build_id: str | None, +) -> list[str]: + """Reasons the manifest must NOT be hand-finished. Empty list means go ahead. + + Every one of these was a real failure mode: writing StateFlags=4 after a + steamcmd run that did nothing turned a dead run into a reported success. + """ + problems: list[str] = [] + + # An empty required set would make the depot check below vacuously true and + # wave through a download that fetched nothing. It means the depot filter + # matched no depot at all (wrong language or OS for this app), not that + # there is nothing to install. + if not required_depots: + problems.append( + "no depots match this app for the chosen language/OS -- refusing to " + "guess that the install is complete" + ) + + cached = recognize.discover_cached_depot_manifests(install_dir) + + # `finish_manifest_manually` drops depots the on-disk manifest lists under + # SharedDepots, which is a different source of truth from app_info's + # depotfromapp/sharedinstall. If they disagree it can end up with nothing to + # install and raise -- and by then clear_steam_staging has already deleted + # Steam's resumable staging tree. Catch the disagreement here, while + # refusing is still free. + shared = app_state.get("SharedDepots") + shared_ids = set(shared) if isinstance(shared, dict) else set() + if cached and not (set(cached) - shared_ids): + problems.append( + "every cached depot is listed under SharedDepots in the existing " + "manifest, so there would be nothing to record as installed" + ) + + missing = sorted(required_depots - set(cached)) + if missing: + problems.append( + f"depot(s) {', '.join(missing)} have no cached manifest under " + f"{install_dir}/{recognize.DEPOTDOWNLOADER_DIR} -- the download did not finish" + ) + + buildid = app_state.get("buildid") + target = app_state.get("TargetBuildID") + if not (buildid and buildid != "0") and not (target and target != "0"): + problems.append( + "manifest has no usable buildid (both buildid and TargetBuildID are 0) " + "-- steamcmd never recorded a real build" + ) + resolved = buildid if buildid and buildid != "0" else target + if expected_build_id is None: + # Don't silently skip the comparison: a None here means app_info had no + # branches.public, so we cannot tell a current build from a stale one. + problems.append( + "could not determine the current public build from app info, so the " + "manifest's buildid cannot be checked" + ) + elif resolved and resolved != expected_build_id: + problems.append( + f"manifest buildid {resolved} != current public build " + f"{expected_build_id} -- content is for a different build" + ) + + # Cache *filenames* are not content. Without this, an install dir holding + # only `.DepotDownloader/_.manifest` and no game files passes every + # other check and gets written as StateFlags=4 with SizeOnDisk=0. + if not manifest_write.measure_content_size(install_dir): + problems.append( + f"no game content found under {install_dir} (only depotdownloader " + "cache files) -- nothing was actually installed" + ) + + pids = recognize.steam_client_pids() + if pids: + problems.append( + f"the Steam client is running (pid {', '.join(str(p) for p in pids)}) " + "and will overwrite the manifest -- close Steam and re-run" + ) + + return problems diff --git a/_installer_import/steam_installer/flow.py b/_installer_import/steam_installer/flow.py index 2fad7e1..eab8e8a 100644 --- a/_installer_import/steam_installer/flow.py +++ b/_installer_import/steam_installer/flow.py @@ -11,8 +11,11 @@ from pathlib import Path from steam_installer import ( appinfo, depotdl, + dlc, + manifest_write, recognize, runlog, + steamcmd_validate, steamctl, takeover, ) @@ -121,18 +124,32 @@ def _download_and_recognize( logger.info("Run log (send this): %s", log.path) return 1 + logger.info("Checking for owned DLC with depots of its own...") + dlc_depots = dlc.download_owned( + metadata.appid, install_target, username, language, args.max_downloads, log + ) + if dlc_depots is None: + logger.info("Run log (send this): %s", log.path) + return 1 + # Downloaded DLC is part of this install now: the finish gate must find its + # depot cached, and the manifest must credit it to its DLC app. + depot_plan.required |= set(dlc_depots) + depot_plan.dlc.update(dlc_depots) + log.section("depotdownloader cache") for depot_id, mid in sorted( recognize.discover_cached_depot_manifests(install_target).items() ): logger.debug("depot %s -> manifest %s", depot_id, mid) - recognize.ensure_installdir_correct(library, metadata.appid, install_target) + manifest_write.ensure_installdir_correct(library, metadata.appid, install_target) if not args.skip_steamcmd: logger.info("Reconciling with steamcmd (validate)...") log.section("steamcmd app_update validate") - result = recognize.run_steamcmd_validate(metadata.appid, username, logger) + result = steamcmd_validate.run_steamcmd_validate( + metadata.appid, username, logger + ) log.file_contents("Manifest after steamcmd", manifest_path) # Never trust steamcmd's exit code alone -- verify the manifest actually diff --git a/_installer_import/steam_installer/ghost.py b/_installer_import/steam_installer/ghost.py index c15ce83..f861225 100644 --- a/_installer_import/steam_installer/ghost.py +++ b/_installer_import/steam_installer/ghost.py @@ -30,7 +30,7 @@ import time from dataclasses import dataclass from pathlib import Path -from steam_installer import recognize, vdf +from steam_installer import manifest_write, recognize, vdf from steam_installer.library import SteamLibrary INSTALLED_STATE_FLAG = 4 @@ -91,7 +91,7 @@ def inspect(library: SteamLibrary, appid: int) -> GhostReport: install_dir = _install_dir(library, app_state) content_bytes = ( - recognize.measure_content_size(install_dir) + manifest_write.measure_content_size(install_dir) if install_dir is not None and install_dir.is_dir() else 0 ) diff --git a/_installer_import/steam_installer/manifest_write.py b/_installer_import/steam_installer/manifest_write.py new file mode 100644 index 0000000..a746021 --- /dev/null +++ b/_installer_import/steam_installer/manifest_write.py @@ -0,0 +1,191 @@ +"""Every write this tool makes to an appmanifest, and only from verified values. + +Split out of recognize.py (250-line cap). When steamcmd's validate cannot +finish the job (see :mod:`steamcmd_validate`), the manifest is finished by hand +using ONLY values already verified: the exact manifest IDs from +depotdownloader's own local cache (never guessed), and the real on-disk size +measured directly. + +Do NOT leave the download bookkeeping for Steam to tidy up on its next launch. +An earlier version did, on the belief that Steam self-corrects; it does not. A +manifest that says StateFlags=4 while `BytesDownloaded` still differs from +`BytesToDownload` is self-contradictory, and Steam resolves the contradiction +by discarding the "installed" claim and resuming the download -- 90 GB of it, +in the run that prompted this comment. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from steam_installer import recognize, vdf +from steam_installer.library import SteamLibrary + + +def ensure_installdir_correct( + library: SteamLibrary, appid: int, install_dir: Path +) -> None: + """Patch a stale `installdir` field BEFORE running steamcmd, not just after. + + Confirmed necessary: a prior, unrelated Steam GUI glitch left this app's + manifest pointing at a different folder name. steamcmd's own validate run + didn't correct it despite a clean exit, and relaunching the Steam client + read that same stale field and wiped the correctly-downloaded folder as + "orphaned" while starting a fresh download into the wrong one. Fixing it + proactively, before steamcmd ever runs, closes that window entirely. + Does nothing if no manifest exists yet -- steamcmd's own app_update writes + one in that case, and it derives installdir from the app's metadata. + """ + manifest_path = library.steamapps / f"appmanifest_{appid}.acf" + if not manifest_path.is_file(): + return + data = vdf.parse(manifest_path.read_text()) + app_state = data.get("AppState", {}) + if not isinstance(app_state, dict): + return + if app_state.get("installdir") == install_dir.name: + return + app_state["installdir"] = install_dir.name + backup_once(manifest_path) + write_atomic(manifest_path, vdf.dumps("AppState", app_state)) + + +def measure_content_size(install_dir: Path) -> int: + """Bytes of real game content, excluding depotdownloader's own cache/staging. + + `.DepotDownloader/` holds manifests and a staging tree that Steam knows + nothing about; counting them overstates SizeOnDisk by gigabytes. + """ + total = 0 + for entry in install_dir.rglob("*"): + if recognize.DEPOTDOWNLOADER_DIR in entry.parts: + continue + if entry.is_file() and not entry.is_symlink(): + total += entry.stat().st_size + return total + + +def finish_manifest_manually( + library: SteamLibrary, + appid: int, + install_dir: Path, + now_unix: int, + dlc_depots: dict[str, str] | None = None, +) -> Path: + """Patch appmanifest_.acf to StateFlags=4 using only verified-known values. + + Requires steamcmd to have already written a partial manifest (even a crashed + run does) -- that's where buildid and SharedDepots come from. This never + fabricates a manifest from nothing. + + `dlc_depots` maps a DLC depot to its DLC app id. Steam's own manifests + record each DLC depot with that `dlcappid`, so it is written the same way. + """ + manifest_path = library.steamapps / f"appmanifest_{appid}.acf" + if not manifest_path.is_file(): + raise recognize.RecognizeError( + f"No existing {manifest_path.name} to finish -- run steamcmd's " + "app_update at least once first (even a crashed run writes one)." + ) + data = vdf.parse(manifest_path.read_text()) + app_state = data.get("AppState", {}) + if not isinstance(app_state, dict): + raise recognize.RecognizeError( + f"{manifest_path} is not a valid AppState manifest" + ) + + depot_manifests = recognize.discover_cached_depot_manifests(install_dir) + if not depot_manifests: + raise recognize.RecognizeError( + f"No cached depot manifests found under {install_dir}/{recognize.DEPOTDOWNLOADER_DIR}" + ) + + size_on_disk = measure_content_size(install_dir) + # "0" means "no real build installed yet" (steamcmd/Steam use it as a + # sentinel) -- a bare `or` treats that non-empty string as truthy and + # never falls through to TargetBuildID, leaving a stale/wrong buildid. + existing_buildid = app_state.get("buildid") + buildid = existing_buildid if existing_buildid and existing_buildid != "0" else None + buildid = buildid or app_state.get("TargetBuildID") or "0" + + # A shared depot (e.g. Steamworks Common Redistributables) is installed once + # under its owning app and only referenced here. Listing it in + # InstalledDepots as though this app owned a copy contradicts SharedDepots. + shared = app_state.get("SharedDepots") + shared_ids = set(shared) if isinstance(shared, dict) else set() + dlc_depots = dlc_depots or {} + installed_depots: dict = {} + for depot_id, manifest_id in depot_manifests.items(): + if depot_id in shared_ids: + continue + entry = {"manifest": manifest_id, "size": "0"} + if depot_id in dlc_depots: + entry["dlcappid"] = dlc_depots[depot_id] + installed_depots[depot_id] = entry + if not installed_depots: + raise recognize.RecognizeError( + f"Every cached depot for {appid} is a shared depot -- nothing to install" + ) + + app_state["StateFlags"] = "4" + app_state["installdir"] = install_dir.name + app_state["LastUpdated"] = str(now_unix) + app_state["SizeOnDisk"] = str(size_on_disk) + app_state["StagingSize"] = "0" + app_state["buildid"] = buildid + app_state["UpdateResult"] = "0" + app_state["InstalledDepots"] = installed_depots + app_state.pop("StagedDepots", None) + # Steam's own per-DLC progress for a download it started: left in place it + # contradicts the settled counters and Steam resumes fetching the DLC. + app_state.pop("DlcDownloads", None) + settle_download_counters(app_state) + + backup_once(manifest_path) + write_atomic(manifest_path, vdf.dumps("AppState", app_state)) + return manifest_path + + +def backup_once(manifest_path: Path) -> None: + """Keep the *pristine* manifest, not the previous run's output. + + An unconditional backup means a second run overwrites the original with + run 1's already-patched file, so the recovery artifact for a 90+ GB install + is one re-run away from being useless. Only write it if absent. + """ + backup_path = manifest_path.with_suffix(".acf.bak") + if not backup_path.exists(): + backup_path.write_text(manifest_path.read_text()) + + +def write_atomic(path: Path, text: str) -> None: + """Write via a temp file + rename so a crash can't truncate the manifest. + + A partial write leaves an unparseable manifest, which Steam and this tool + both then choke on. `os.replace` is atomic within a filesystem, and the + temp file is created alongside the target to stay on the same one. + """ + tmp = path.with_name(f"{path.name}.tmp{os.getpid()}") + try: + tmp.write_text(text) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def settle_download_counters(app_state: dict) -> None: + """Mark the download bookkeeping complete: `Downloaded` == `ToDownload`. + + Leaving these at their mid-download values is what made Steam discard a + StateFlags=4 manifest and resume fetching. Real installed manifests keep + these fields (they are not dropped) with the pair values equal. + """ + for total_key, done_key in ( + ("BytesToDownload", "BytesDownloaded"), + ("BytesToStage", "BytesStaged"), + ): + total = app_state.get(total_key) + if total is not None: + app_state[done_key] = total diff --git a/_installer_import/steam_installer/recognize.py b/_installer_import/steam_installer/recognize.py index 59fd033..25f6df6 100644 --- a/_installer_import/steam_installer/recognize.py +++ b/_installer_import/steam_installer/recognize.py @@ -1,43 +1,19 @@ -"""Make the Steam client recognize downloaded content as installed. +"""Read-only questions about whether Steam will see an install as real. -Primary path: `steamcmd ... app_update validate`. It hash-checks -existing files against the depot manifest and only fetches real deltas -- -confirmed live against a real 90+ GB install, going from ~24% to ~41% -credited in one pass without redownloading from scratch. - -Fallback: some titles ship a post-install script (installscript.vdf) that -steamcmd tries to run through a private, headless Proton setup. On at least -one real title this hit a confirmed steamcmd bug -- a stack-overflow segfault -in the closed-source steamclient.so error-handling path after the script -evaluator reports failure (crash backtrace: the same stack frame repeating -dozens of times). There's no known fix. When that specific crash is -detected, we finish the appmanifest by hand using ONLY values we already -verified ourselves: the exact manifest IDs from depotdownloader's own local -cache (never guessed), and the real on-disk size measured directly. - -Do NOT leave the download bookkeeping for Steam to tidy up on its next -launch. An earlier version of this module did, on the belief that Steam -self-corrects; it does not. A manifest that says StateFlags=4 while -`BytesDownloaded` still differs from `BytesToDownload` is self-contradictory, -and Steam resolves the contradiction by discarding the "installed" claim and -resuming the download -- 90 GB of it, in the run that prompted this comment. -The byte counters (`_settle_download_counters`) and the staging tree -(`clear_steam_staging`) must both be settled here, and -`manifest_reflects_real_install` refuses to call an install real until they -are. +Split on 2026-09-25 (250-line cap): steamcmd's validate run now lives in +:mod:`steamcmd_validate`, every manifest write in :mod:`manifest_write`, and +the preconditions for writing in :mod:`finish_check`. What stays here is the +shared vocabulary they all read: depotdownloader's cache, Steam's staging tree +and client process, and whether a manifest's bookkeeping says "done". """ from __future__ import annotations -import logging -import os import re import shutil -import signal -from dataclasses import dataclass from pathlib import Path -from steam_installer import runlog, vdf +from steam_installer import vdf from steam_installer.library import SteamLibrary MANIFEST_CACHE_RE = re.compile(r"^(\d+)_(\d+)\.manifest$") @@ -54,90 +30,6 @@ class RecognizeError(Exception): pass -@dataclass -class ValidateResult: - completed: bool - crashed_on_install_script: bool - returncode: int - output: str = "" - - -def run_steamcmd_validate( - appid: int, - username: str, - logger: logging.Logger, -) -> ValidateResult: - """Run steamcmd's app_update ... validate. - - Takes no install dir on purpose. `force_install_dir` sets a Steam *library - root*, not an install directory: pointing it at the game folder made - steamcmd look for manifests in `/steamapps/`, find none ("Loaded 0 - apps from install folder"), create that stray directory, and then fail the - app with "Invalid platform" because it had no depot context for it. The app - already lives in a registered library, so letting steamcmd use the real one - is both correct and sufficient. - - Output is teed through a pty (see runlog) rather than captured: steamcmd - prompts for a password and possibly a Steam Guard code on stdin, and prompt - text written to a captured/buffered pipe never reaches the terminal -- the - process looks hung with no visible request for input. - - The confirmed steamcmd install-script bug (see module docstring) kills - the process with SIGSEGV -- Python reports that as returncode -11 on - POSIX. Checking the signal directly is more reliable than grepping - captured text for a message string. - """ - cmd = [ - "steamcmd", - # A Linux steamcmd refuses a Windows-only app with "Invalid platform" - # unless told which platform to resolve depots for. Must precede +login. - "+@sSteamCmdForcePlatformType", - "windows", - "+login", - username, - "+app_update", - str(appid), - "validate", - "+quit", - ] - returncode, output = runlog.run_logged(cmd, logger, timeout=3600) - crashed = returncode == -signal.SIGSEGV - return ValidateResult( - completed=returncode == 0, - crashed_on_install_script=crashed, - returncode=returncode, - output=output, - ) - - -def ensure_installdir_correct( - library: SteamLibrary, appid: int, install_dir: Path -) -> None: - """Patch a stale `installdir` field BEFORE running steamcmd, not just after. - - Confirmed necessary: a prior, unrelated Steam GUI glitch left this app's - manifest pointing at a different folder name. steamcmd's own validate run - didn't correct it despite a clean exit, and relaunching the Steam client - read that same stale field and wiped the correctly-downloaded folder as - "orphaned" while starting a fresh download into the wrong one. Fixing it - proactively, before steamcmd ever runs, closes that window entirely. - Does nothing if no manifest exists yet -- steamcmd's own app_update writes - one in that case, and it derives installdir from the app's metadata. - """ - manifest_path = library.steamapps / f"appmanifest_{appid}.acf" - if not manifest_path.is_file(): - return - data = vdf.parse(manifest_path.read_text()) - app_state = data.get("AppState", {}) - if not isinstance(app_state, dict): - return - if app_state.get("installdir") == install_dir.name: - return - app_state["installdir"] = install_dir.name - _backup_once(manifest_path) - _write_atomic(manifest_path, vdf.dumps("AppState", app_state)) - - def manifest_reflects_real_install( library: SteamLibrary, appid: int, install_dir: Path ) -> bool: @@ -258,216 +150,3 @@ def steam_client_pids() -> list[int]: if exe.name in ("steam", "steamwebhelper"): pids.append(int(entry.name)) return pids - - -def check_ready_to_finish( - install_dir: Path, - app_state: dict, - required_depots: set[str], - expected_build_id: str | None, -) -> list[str]: - """Reasons the manifest must NOT be hand-finished. Empty list means go ahead. - - Every one of these was a real failure mode: writing StateFlags=4 after a - steamcmd run that did nothing turned a dead run into a reported success. - """ - problems: list[str] = [] - - # An empty required set would make the depot check below vacuously true and - # wave through a download that fetched nothing. It means the depot filter - # matched no depot at all (wrong language or OS for this app), not that - # there is nothing to install. - if not required_depots: - problems.append( - "no depots match this app for the chosen language/OS -- refusing to " - "guess that the install is complete" - ) - - cached = discover_cached_depot_manifests(install_dir) - - # `finish_manifest_manually` drops depots the on-disk manifest lists under - # SharedDepots, which is a different source of truth from app_info's - # depotfromapp/sharedinstall. If they disagree it can end up with nothing to - # install and raise -- and by then clear_steam_staging has already deleted - # Steam's resumable staging tree. Catch the disagreement here, while - # refusing is still free. - shared = app_state.get("SharedDepots") - shared_ids = set(shared) if isinstance(shared, dict) else set() - if cached and not (set(cached) - shared_ids): - problems.append( - "every cached depot is listed under SharedDepots in the existing " - "manifest, so there would be nothing to record as installed" - ) - - missing = sorted(required_depots - set(cached)) - if missing: - problems.append( - f"depot(s) {', '.join(missing)} have no cached manifest under " - f"{install_dir}/{DEPOTDOWNLOADER_DIR} -- the download did not finish" - ) - - buildid = app_state.get("buildid") - target = app_state.get("TargetBuildID") - if not (buildid and buildid != "0") and not (target and target != "0"): - problems.append( - "manifest has no usable buildid (both buildid and TargetBuildID are 0) " - "-- steamcmd never recorded a real build" - ) - resolved = buildid if buildid and buildid != "0" else target - if expected_build_id is None: - # Don't silently skip the comparison: a None here means app_info had no - # branches.public, so we cannot tell a current build from a stale one. - problems.append( - "could not determine the current public build from app info, so the " - "manifest's buildid cannot be checked" - ) - elif resolved and resolved != expected_build_id: - problems.append( - f"manifest buildid {resolved} != current public build " - f"{expected_build_id} -- content is for a different build" - ) - - # Cache *filenames* are not content. Without this, an install dir holding - # only `.DepotDownloader/_.manifest` and no game files passes every - # other check and gets written as StateFlags=4 with SizeOnDisk=0. - if not measure_content_size(install_dir): - problems.append( - f"no game content found under {install_dir} (only depotdownloader " - "cache files) -- nothing was actually installed" - ) - - pids = steam_client_pids() - if pids: - problems.append( - f"the Steam client is running (pid {', '.join(str(p) for p in pids)}) " - "and will overwrite the manifest -- close Steam and re-run" - ) - - return problems - - -def measure_content_size(install_dir: Path) -> int: - """Bytes of real game content, excluding depotdownloader's own cache/staging. - - `.DepotDownloader/` holds manifests and a staging tree that Steam knows - nothing about; counting them overstates SizeOnDisk by gigabytes. - """ - total = 0 - for entry in install_dir.rglob("*"): - if DEPOTDOWNLOADER_DIR in entry.parts: - continue - if entry.is_file() and not entry.is_symlink(): - total += entry.stat().st_size - return total - - -def finish_manifest_manually( - library: SteamLibrary, - appid: int, - install_dir: Path, - now_unix: int, -) -> Path: - """Patch appmanifest_.acf to StateFlags=4 using only verified-known values. - - Requires steamcmd to have already written a partial manifest (even a crashed - run does) -- that's where buildid and SharedDepots come from. This never - fabricates a manifest from nothing. - """ - manifest_path = library.steamapps / f"appmanifest_{appid}.acf" - if not manifest_path.is_file(): - raise RecognizeError( - f"No existing {manifest_path.name} to finish -- run steamcmd's " - "app_update at least once first (even a crashed run writes one)." - ) - data = vdf.parse(manifest_path.read_text()) - app_state = data.get("AppState", {}) - if not isinstance(app_state, dict): - raise RecognizeError(f"{manifest_path} is not a valid AppState manifest") - - depot_manifests = discover_cached_depot_manifests(install_dir) - if not depot_manifests: - raise RecognizeError( - f"No cached depot manifests found under {install_dir}/{DEPOTDOWNLOADER_DIR}" - ) - - size_on_disk = measure_content_size(install_dir) - # "0" means "no real build installed yet" (steamcmd/Steam use it as a - # sentinel) -- a bare `or` treats that non-empty string as truthy and - # never falls through to TargetBuildID, leaving a stale/wrong buildid. - existing_buildid = app_state.get("buildid") - buildid = existing_buildid if existing_buildid and existing_buildid != "0" else None - buildid = buildid or app_state.get("TargetBuildID") or "0" - - # A shared depot (e.g. Steamworks Common Redistributables) is installed once - # under its owning app and only referenced here. Listing it in - # InstalledDepots as though this app owned a copy contradicts SharedDepots. - shared = app_state.get("SharedDepots") - shared_ids = set(shared) if isinstance(shared, dict) else set() - installed_depots = { - depot_id: {"manifest": manifest_id, "size": "0"} - for depot_id, manifest_id in depot_manifests.items() - if depot_id not in shared_ids - } - if not installed_depots: - raise RecognizeError( - f"Every cached depot for {appid} is a shared depot -- nothing to install" - ) - - app_state["StateFlags"] = "4" - app_state["installdir"] = install_dir.name - app_state["LastUpdated"] = str(now_unix) - app_state["SizeOnDisk"] = str(size_on_disk) - app_state["StagingSize"] = "0" - app_state["buildid"] = buildid - app_state["UpdateResult"] = "0" - app_state["InstalledDepots"] = installed_depots - app_state.pop("StagedDepots", None) - _settle_download_counters(app_state) - - _backup_once(manifest_path) - _write_atomic(manifest_path, vdf.dumps("AppState", app_state)) - return manifest_path - - -def _backup_once(manifest_path: Path) -> None: - """Keep the *pristine* manifest, not the previous run's output. - - An unconditional backup means a second run overwrites the original with - run 1's already-patched file, so the recovery artifact for a 90+ GB install - is one re-run away from being useless. Only write it if absent. - """ - backup_path = manifest_path.with_suffix(".acf.bak") - if not backup_path.exists(): - backup_path.write_text(manifest_path.read_text()) - - -def _write_atomic(path: Path, text: str) -> None: - """Write via a temp file + rename so a crash can't truncate the manifest. - - A partial write leaves an unparseable manifest, which Steam and this tool - both then choke on. `os.replace` is atomic within a filesystem, and the - temp file is created alongside the target to stay on the same one. - """ - tmp = path.with_name(f"{path.name}.tmp{os.getpid()}") - try: - tmp.write_text(text) - os.replace(tmp, path) - except BaseException: - tmp.unlink(missing_ok=True) - raise - - -def _settle_download_counters(app_state: dict) -> None: - """Mark the download bookkeeping complete: `Downloaded` == `ToDownload`. - - Leaving these at their mid-download values is what made Steam discard a - StateFlags=4 manifest and resume fetching. Real installed manifests keep - these fields (they are not dropped) with the pair values equal. - """ - for total_key, done_key in ( - ("BytesToDownload", "BytesDownloaded"), - ("BytesToStage", "BytesStaged"), - ): - total = app_state.get(total_key) - if total is not None: - app_state[done_key] = total diff --git a/_installer_import/steam_installer/steamcmd_validate.py b/_installer_import/steam_installer/steamcmd_validate.py new file mode 100644 index 0000000..b62f70d --- /dev/null +++ b/_installer_import/steam_installer/steamcmd_validate.py @@ -0,0 +1,79 @@ +"""Reconcile an install with Steam's own bookkeeping via `steamcmd ... validate`. + +Split out of recognize.py (250-line cap). `app_update validate` hash-checks +existing files against the depot manifest and only fetches real deltas -- +confirmed live against a real 90+ GB install, going from ~24% to ~41% credited +in one pass without redownloading from scratch. + +Some titles ship a post-install script (installscript.vdf) that steamcmd runs +through a private, headless Proton setup. On at least one real title this hit a +confirmed steamcmd bug -- a stack-overflow segfault in the closed-source +steamclient.so error-handling path after the script evaluator reports failure +(the same stack frame repeating dozens of times). There's no known fix; the +caller detects that crash and finishes the manifest by hand instead +(:mod:`manifest_write`). +""" + +from __future__ import annotations + +import logging +import signal +from dataclasses import dataclass + +from steam_installer import runlog + + +@dataclass +class ValidateResult: + completed: bool + crashed_on_install_script: bool + returncode: int + output: str = "" + + +def run_steamcmd_validate( + appid: int, + username: str, + logger: logging.Logger, +) -> ValidateResult: + """Run steamcmd's app_update ... validate. + + Takes no install dir on purpose. `force_install_dir` sets a Steam *library + root*, not an install directory: pointing it at the game folder made + steamcmd look for manifests in `/steamapps/`, find none ("Loaded 0 + apps from install folder"), create that stray directory, and then fail the + app with "Invalid platform" because it had no depot context for it. The app + already lives in a registered library, so letting steamcmd use the real one + is both correct and sufficient. + + Output is teed through a pty (see runlog) rather than captured: steamcmd + prompts for a password and possibly a Steam Guard code on stdin, and prompt + text written to a captured/buffered pipe never reaches the terminal -- the + process looks hung with no visible request for input. + + The confirmed steamcmd install-script bug (see module docstring) kills + the process with SIGSEGV -- Python reports that as returncode -11 on + POSIX. Checking the signal directly is more reliable than grepping + captured text for a message string. + """ + cmd = [ + "steamcmd", + # A Linux steamcmd refuses a Windows-only app with "Invalid platform" + # unless told which platform to resolve depots for. Must precede +login. + "+@sSteamCmdForcePlatformType", + "windows", + "+login", + username, + "+app_update", + str(appid), + "validate", + "+quit", + ] + returncode, output = runlog.run_logged(cmd, logger, timeout=3600) + crashed = returncode == -signal.SIGSEGV + return ValidateResult( + completed=returncode == 0, + crashed_on_install_script=crashed, + returncode=returncode, + output=output, + ) diff --git a/_installer_import/tests/_cli_helpers.py b/_installer_import/tests/_cli_helpers.py new file mode 100644 index 0000000..d9ce157 --- /dev/null +++ b/_installer_import/tests/_cli_helpers.py @@ -0,0 +1,21 @@ +"""Stubs shared by the CLI flow tests (plain functions, not fixtures).""" + +from steam_installer import depotdl, steamcmd_validate + +ARGV = ["--appid", "1771300", "--username", "u", "--language", "english"] + + +def _stub_download(monkeypatch, returncode=0): + monkeypatch.setattr(depotdl, "download", lambda **kw: returncode) + + +def _stub_steamcmd(monkeypatch, returncode, crashed=False): + monkeypatch.setattr( + steamcmd_validate, + "run_steamcmd_validate", + lambda appid, username, logger: steamcmd_validate.ValidateResult( + completed=returncode == 0, + crashed_on_install_script=crashed, + returncode=returncode, + ), + ) diff --git a/_installer_import/tests/_recognize_helpers.py b/_installer_import/tests/_recognize_helpers.py new file mode 100644 index 0000000..7af8a1d --- /dev/null +++ b/_installer_import/tests/_recognize_helpers.py @@ -0,0 +1,31 @@ +"""Fixtures-as-functions shared by the recognize/manifest test files.""" + +from pathlib import Path + +from steam_installer import vdf + + +def _write_partial_manifest(steamapps: Path, appid: int) -> None: + steamapps.mkdir(parents=True, exist_ok=True) + data = { + "appid": str(appid), + "StateFlags": "1026", + "buildid": "23914554", + "SharedDepots": {"228989": "228980"}, + } + (steamapps / f"appmanifest_{appid}.acf").write_text(vdf.dumps("AppState", data)) + + +def _fake_proc(tmp_path, entries): + """Build a /proc-shaped tree: {pid: exe_target_path_or_None}.""" + proc = tmp_path / "proc" + proc.mkdir() + for name, target in entries.items(): + entry = proc / name + entry.mkdir() + if target is not None: + real = tmp_path / "bin" / target + real.parent.mkdir(parents=True, exist_ok=True) + real.write_text("") + (entry / "exe").symlink_to(real) + return proc diff --git a/_installer_import/tests/conftest.py b/_installer_import/tests/conftest.py index 3757bdf..0956df6 100644 --- a/_installer_import/tests/conftest.py +++ b/_installer_import/tests/conftest.py @@ -14,7 +14,6 @@ import os import pathlib import pytest - from steam_installer import ( appinfo, cli, @@ -131,3 +130,67 @@ def cli_steam(tmp_path, monkeypatch, isolated_cli): depotdl, "download", lambda **kw: pytest.fail("must not download") ) return library + + +@pytest.fixture(autouse=True) +def _no_host_depotdownloader(tmp_path, monkeypatch): + """Never let this machine's ~/.local/bin decide which binary a test sees.""" + monkeypatch.setattr(deps, "MANAGED_DEPOTDOWNLOADER", tmp_path / "absent") + + +@pytest.fixture +def cli_logs(tmp_path, monkeypatch): + monkeypatch.setattr(runlog, "LOG_DIR", tmp_path / "state") + yield + logger = logging.getLogger(runlog.LOGGER_NAME) + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + +@pytest.fixture +def steam(tmp_path, monkeypatch): + """A Steam tree with KCD2 downloaded but its manifest not yet finished.""" + root = tmp_path / "Steam" + library = SteamLibrary(root) + install_dir = library.common / "KingdomComeDeliverance2" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + library.steamapps.mkdir(parents=True, exist_ok=True) + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps( + "AppState", + { + "appid": "1771300", + "StateFlags": "1026", + "buildid": "23914554", + "BytesToDownload": "92009052960", + "BytesDownloaded": "22129058224", + }, + ) + ) + + monkeypatch.setattr(cli, "find_steam_root", lambda: root) + monkeypatch.setattr(cli, "list_libraries", lambda r: [library]) + monkeypatch.setattr(cli, "pick_install_target", lambda libs, d: install_dir) + monkeypatch.setattr(deps.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr( + appinfo, + "get_metadata", + lambda appid: appinfo.AppMetadata(appid, "KCD2", "KingdomComeDeliverance2"), + ) + monkeypatch.setattr( + appinfo, + "get_depot_plan", + lambda appid, language: appinfo.DepotPlan({"1771303"}, {"228989"}), + ) + monkeypatch.setattr(appinfo, "get_build_id", lambda appid: "23914554") + # No DLC unless a test says so: the real lookup is a steamcmd call. + monkeypatch.setattr(appinfo, "get_dlc_plans", lambda appid, language: {}) + monkeypatch.setattr(recognize, "steam_client_pids", lambda: []) + monkeypatch.setattr("builtins.input", lambda *a: "y") + # Prompting is tty-gated now, and pytest's stdin is not a tty. + monkeypatch.setattr(cli, "_is_interactive", lambda args: True) + return library, install_dir diff --git a/_installer_import/tests/test_cli.py b/_installer_import/tests/test_cli.py index db0aef3..881c92b 100644 --- a/_installer_import/tests/test_cli.py +++ b/_installer_import/tests/test_cli.py @@ -6,90 +6,20 @@ StateFlags=4, printed "launch Steam and confirm Play works" and returned 0. These tests pin the fail-closed behaviour. """ -import logging -import subprocess -import sys - import pytest +from steam_installer import ( + appinfo, + cli, + depotdl, + manifest_write, + recognize, + steamcmd_validate, + vdf, +) -from steam_installer import appinfo, cli, depotdl, deps, recognize, runlog, vdf -from steam_installer.library import SteamLibrary +from _cli_helpers import ARGV, _stub_download, _stub_steamcmd - -@pytest.fixture(autouse=True) -def isolated_logs(tmp_path, monkeypatch): - monkeypatch.setattr(runlog, "LOG_DIR", tmp_path / "state") - yield - logger = logging.getLogger(runlog.LOGGER_NAME) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - -@pytest.fixture -def steam(tmp_path, monkeypatch): - """A Steam tree with KCD2 downloaded but its manifest not yet finished.""" - root = tmp_path / "Steam" - library = SteamLibrary(root) - install_dir = library.common / "KingdomComeDeliverance2" - cache = install_dir / ".DepotDownloader" - cache.mkdir(parents=True) - (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") - (install_dir / "game.pak").write_bytes(b"x" * 1000) - library.steamapps.mkdir(parents=True, exist_ok=True) - (library.steamapps / "appmanifest_1771300.acf").write_text( - vdf.dumps( - "AppState", - { - "appid": "1771300", - "StateFlags": "1026", - "buildid": "23914554", - "BytesToDownload": "92009052960", - "BytesDownloaded": "22129058224", - }, - ) - ) - - monkeypatch.setattr(cli, "find_steam_root", lambda: root) - monkeypatch.setattr(cli, "list_libraries", lambda r: [library]) - monkeypatch.setattr(cli, "pick_install_target", lambda libs, d: install_dir) - monkeypatch.setattr(deps.shutil, "which", lambda name: f"/usr/bin/{name}") - monkeypatch.setattr( - appinfo, - "get_metadata", - lambda appid: appinfo.AppMetadata(appid, "KCD2", "KingdomComeDeliverance2"), - ) - monkeypatch.setattr( - appinfo, - "get_depot_plan", - lambda appid, language: appinfo.DepotPlan({"1771303"}, {"228989"}), - ) - monkeypatch.setattr(appinfo, "get_build_id", lambda appid: "23914554") - monkeypatch.setattr(recognize, "steam_client_pids", lambda: []) - monkeypatch.setattr(recognize, "steam_client_pids", lambda: []) - monkeypatch.setattr("builtins.input", lambda *a: "y") - # Prompting is tty-gated now, and pytest's stdin is not a tty. - monkeypatch.setattr(cli, "_is_interactive", lambda args: True) - return library, install_dir - - -ARGV = ["--appid", "1771300", "--username", "u", "--language", "english"] - - -def _stub_download(monkeypatch, returncode=0): - monkeypatch.setattr(depotdl, "download", lambda **kw: returncode) - - -def _stub_steamcmd(monkeypatch, returncode, crashed=False): - monkeypatch.setattr( - recognize, - "run_steamcmd_validate", - lambda appid, username, logger: recognize.ValidateResult( - completed=returncode == 0, - crashed_on_install_script=crashed, - returncode=returncode, - ), - ) +pytestmark = pytest.mark.usefixtures("cli_logs") def test_dead_steamcmd_run_still_finishes_when_content_is_present(steam, monkeypatch): @@ -167,12 +97,12 @@ def test_clean_steamcmd_run_short_circuits(steam, monkeypatch): }, ) ) - return recognize.ValidateResult(True, False, 0) + return steamcmd_validate.ValidateResult(True, False, 0) - monkeypatch.setattr(recognize, "run_steamcmd_validate", fake_validate) + monkeypatch.setattr(steamcmd_validate, "run_steamcmd_validate", fake_validate) called = [] monkeypatch.setattr( - recognize, + manifest_write, "finish_manifest_manually", lambda **kw: called.append(1), ) @@ -207,12 +137,12 @@ def test_a_complete_install_after_a_failed_steamcmd_is_not_credited_to_it( }, ) ) - return recognize.ValidateResult(True, False, 5) + return steamcmd_validate.ValidateResult(True, False, 5) - monkeypatch.setattr(recognize, "run_steamcmd_validate", fake_validate) + monkeypatch.setattr(steamcmd_validate, "run_steamcmd_validate", fake_validate) called = [] monkeypatch.setattr( - recognize, "finish_manifest_manually", lambda **kw: called.append(1) + manifest_write, "finish_manifest_manually", lambda **kw: called.append(1) ) assert cli.main(ARGV) == 0 @@ -243,106 +173,3 @@ def test_unhandled_error_returns_nonzero_and_names_the_log(steam, monkeypatch, c def test_requires_a_name_or_appid(): with pytest.raises(SystemExit): cli.main(["--username", "u"]) - - -def test_missing_manifest_under_skip_steamcmd_does_not_blame_steamcmd( - steam, monkeypatch, capsys -): - """--skip-steamcmd never runs steamcmd, so "steamcmd never ran far enough" - would send the user chasing the wrong thing.""" - library, _ = steam - (library.steamapps / "appmanifest_1771300.acf").unlink() - _stub_download(monkeypatch) - - assert cli.main([*ARGV, "--skip-steamcmd"]) == 1 - - out = capsys.readouterr().out - assert "never fabricates one" in out - assert "Re-run without --skip-steamcmd" in out - assert "steamcmd never ran far enough" not in out - - -def test_missing_manifest_with_steamcmd_blames_steamcmd(steam, monkeypatch, capsys): - library, _ = steam - (library.steamapps / "appmanifest_1771300.acf").unlink() - _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) - - assert cli.main(ARGV) == 1 - assert "steamcmd never ran far enough" in capsys.readouterr().out - - -def test_language_defaults_to_english_when_prompt_is_empty(steam, monkeypatch): - seen = {} - monkeypatch.setattr("builtins.input", lambda *a: "y" if "Proceed" in a[0] else "") - monkeypatch.setattr( - appinfo, - "get_depot_plan", - lambda appid, language: ( - seen.setdefault("language", language) - and None - or appinfo.DepotPlan({"1771303"}, set()) - ), - ) - _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) - - cli.main(["--appid", "1771300", "--username", "u"]) - - assert seen["language"] == "english" - - -def test_reports_the_known_install_script_crash(steam, monkeypatch, capsys): - _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=-11, crashed=True) - - assert cli.main(ARGV) == 0 - assert "known install-script crash" in capsys.readouterr().out - - -def test_adopts_staged_content_then_clears_it(steam, monkeypatch, capsys): - """Steam's staging tree is adopted into the install dir before downloading, - and only cleared once the install is finished -- never discarded up front.""" - library, install_dir = steam - staged = library.steamapps / "downloading" / "1771300" - staged.mkdir(parents=True) - (staged / "big.pak").write_bytes(b"x" * 2_000_000) - _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) - - assert cli.main(ARGV) == 0 - - out = capsys.readouterr().out - assert "Hardlinking it" in out - assert "now-redundant Steam staging data" in out - # Adopted, not copied: the file is one inode with two names. - assert (install_dir / "big.pak").stat().st_size == 2_000_000 - - -def test_fails_when_written_manifest_does_not_read_back_as_installed( - steam, monkeypatch, capsys -): - """Claiming success without re-reading the write is the same fail-open - mistake in a different place.""" - _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) - monkeypatch.setattr( - recognize, "manifest_reflects_real_install", lambda *a, **kw: False - ) - - assert cli.main(ARGV) == 1 - assert "does not read back as a real install" in capsys.readouterr().out - - -def test_module_is_runnable_as_a_script(coverage_subprocess_env): - """`python -m steam_installer.cli` is a real entry point (the __main__ - guard), so it must not be dead code.""" - result = subprocess.run( - [sys.executable, "-m", "steam_installer.cli", "--help"], - capture_output=True, - text=True, - timeout=60, - env=coverage_subprocess_env, - ) - assert result.returncode == 0 - assert "--skip-steamcmd" in result.stdout diff --git a/_installer_import/tests/test_cli_finish.py b/_installer_import/tests/test_cli_finish.py new file mode 100644 index 0000000..29a2d06 --- /dev/null +++ b/_installer_import/tests/test_cli_finish.py @@ -0,0 +1,118 @@ +"""CLI flow tests, part 2: the manifest-finishing end of a run.""" + +import subprocess +import sys + +import pytest +from steam_installer import ( + appinfo, + cli, + recognize, +) + +from _cli_helpers import ARGV, _stub_download, _stub_steamcmd + +pytestmark = pytest.mark.usefixtures("cli_logs") + + +def test_missing_manifest_under_skip_steamcmd_does_not_blame_steamcmd( + steam, monkeypatch, capsys +): + """--skip-steamcmd never runs steamcmd, so "steamcmd never ran far enough" + would send the user chasing the wrong thing.""" + library, _ = steam + (library.steamapps / "appmanifest_1771300.acf").unlink() + _stub_download(monkeypatch) + + assert cli.main([*ARGV, "--skip-steamcmd"]) == 1 + + out = capsys.readouterr().out + assert "never fabricates one" in out + assert "Re-run without --skip-steamcmd" in out + assert "steamcmd never ran far enough" not in out + + +def test_missing_manifest_with_steamcmd_blames_steamcmd(steam, monkeypatch, capsys): + library, _ = steam + (library.steamapps / "appmanifest_1771300.acf").unlink() + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + + assert cli.main(ARGV) == 1 + assert "steamcmd never ran far enough" in capsys.readouterr().out + + +def test_language_defaults_to_english_when_prompt_is_empty(steam, monkeypatch): + seen = {} + monkeypatch.setattr("builtins.input", lambda *a: "y" if "Proceed" in a[0] else "") + monkeypatch.setattr( + appinfo, + "get_depot_plan", + lambda appid, language: ( + seen.setdefault("language", language) + and None + or appinfo.DepotPlan({"1771303"}, set()) + ), + ) + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + + cli.main(["--appid", "1771300", "--username", "u"]) + + assert seen["language"] == "english" + + +def test_reports_the_known_install_script_crash(steam, monkeypatch, capsys): + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=-11, crashed=True) + + assert cli.main(ARGV) == 0 + assert "known install-script crash" in capsys.readouterr().out + + +def test_adopts_staged_content_then_clears_it(steam, monkeypatch, capsys): + """Steam's staging tree is adopted into the install dir before downloading, + and only cleared once the install is finished -- never discarded up front.""" + library, install_dir = steam + staged = library.steamapps / "downloading" / "1771300" + staged.mkdir(parents=True) + (staged / "big.pak").write_bytes(b"x" * 2_000_000) + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + + assert cli.main(ARGV) == 0 + + out = capsys.readouterr().out + assert "Hardlinking it" in out + assert "now-redundant Steam staging data" in out + # Adopted, not copied: the file is one inode with two names. + assert (install_dir / "big.pak").stat().st_size == 2_000_000 + + +def test_fails_when_written_manifest_does_not_read_back_as_installed( + steam, monkeypatch, capsys +): + """Claiming success without re-reading the write is the same fail-open + mistake in a different place.""" + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + monkeypatch.setattr( + recognize, "manifest_reflects_real_install", lambda *a, **kw: False + ) + + assert cli.main(ARGV) == 1 + assert "does not read back as a real install" in capsys.readouterr().out + + +def test_module_is_runnable_as_a_script(coverage_subprocess_env): + """`python -m steam_installer.cli` is a real entry point (the __main__ + guard), so it must not be dead code.""" + result = subprocess.run( + [sys.executable, "-m", "steam_installer.cli", "--help"], + capture_output=True, + text=True, + timeout=60, + env=coverage_subprocess_env, + ) + assert result.returncode == 0 + assert "--skip-steamcmd" in result.stdout diff --git a/_installer_import/tests/test_dlc.py b/_installer_import/tests/test_dlc.py new file mode 100644 index 0000000..2b49351 --- /dev/null +++ b/_installer_import/tests/test_dlc.py @@ -0,0 +1,184 @@ +"""DLC that ships as its own app: discovery, download, and crediting it in the manifest. + +Shapes are taken from the real `app_info_print` of Cyberpunk 2077 (1091500) and +its DLC, 2025-09-25: Phantom Liberty (2138330) keeps its ~25 GB depot under its +own app id; 2060310 is listed in the game's own depots; 2441600 has none. +""" + +import logging +import subprocess + +import pytest +from _cli_helpers import ARGV, _stub_download, _stub_steamcmd + +from steam_installer import appinfo, cli, depotdl, deps, dlc, recognize, runlog, vdf + +pytestmark = pytest.mark.usefixtures("cli_logs") + +GAME = { + "extended": {"listofdlc": "2060310,2138330,2441600"}, + "depots": { + "1091501": {"config": {"oslist": "windows"}}, + "2060310": {"dlcappid": "2060310", "config": {"oslist": "windows"}}, + }, +} +DLC_BLOCKS = { + # Listed in the game's own depots: the base run already covers it. + 2060310: {"depots": {"2060310": {"dlcappid": "2060310"}}}, + # Its own depot, tagged with its own id -- exactly what it installs. + 2138330: { + "depots": { + "2138330": {"dlcappid": "2138330", "config": {"oslist": "windows"}}, + "2138331": {"config": {"oslist": "windows", "language": "german"}}, + } + }, + 2441600: {"common": {"name": "Pre-Order Bonus"}}, +} + + +def _app_info(monkeypatch, game=GAME, dlc_blocks=DLC_BLOCKS): + monkeypatch.setattr(appinfo, "_fetch_app_block", lambda appid: game) + monkeypatch.setattr( + appinfo, "_fetch_app_blocks", lambda ids: {i: dlc_blocks[i] for i in ids} + ) + + +def test_only_dlc_with_depots_of_its_own_needs_a_run(monkeypatch): + _app_info(monkeypatch) + assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330"}} + + +@pytest.mark.parametrize("game", [{}, {"extended": "scalar"}, {"extended": {}}]) +def test_a_game_without_dlc_needs_no_lookup(monkeypatch, game): + monkeypatch.setattr(appinfo, "_fetch_app_block", lambda appid: game) + monkeypatch.setattr(appinfo, "_fetch_app_blocks", lambda ids: pytest.fail()) + assert appinfo.get_dlc_plans(1) == {} + + +def test_a_game_with_no_depot_list_still_finds_its_dlc(monkeypatch): + _app_info(monkeypatch, game={"extended": {"listofdlc": "2138330"}}) + assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330"}} + + +def test_depot_plan_tags_the_games_own_dlc_depots(monkeypatch): + _app_info(monkeypatch) + plan = appinfo.get_depot_plan(1091500) + assert plan.required == {"1091501"} + assert plan.dlc == {"2060310": "2060310"} + + +def test_app_info_for_many_apps_comes_from_one_steamcmd_call(monkeypatch): + output = ( + 'Loading...\n"2060310"\n{\n\t"common"\n\t{\n\t\t"name"\t"A"\n\t}\n}\n' + '"2138330"\n{\n\t"common"\n\t{\n\t\t"name"\t"B"\n\t}\n}\nUnloading\n' + ) + calls = [] + + def fake_run(cmd, **kwargs): + calls.append(cmd) + return subprocess.CompletedProcess(cmd, 0, stdout=output, stderr="") + + monkeypatch.setattr(appinfo.subprocess, "run", fake_run) + blocks = appinfo._fetch_app_blocks([2138330, 2060310]) + assert blocks[2060310]["common"]["name"] == "A" + assert blocks[2138330]["common"]["name"] == "B" + assert len(calls) == 1 + assert calls[0].count("+app_info_print") == 2 + + +def _logged(monkeypatch, returncode, output=""): + monkeypatch.setattr( + depotdl.runlog, "run_logged", lambda cmd, logger: (returncode, output) + ) + + +@pytest.mark.parametrize( + ("returncode", "output", "expected"), + [ + (0, "", True), + (1, "App 2441600 (Bonus) is not available from this account.", None), + (1, "Connection lost", False), + ], +) +def test_download_dlc_separates_not_owned_from_failed( + tmp_path, monkeypatch, returncode, output, expected +): + _logged(monkeypatch, returncode, output) + result = depotdl.download_dlc(2441600, tmp_path, "u", logging.getLogger("t")) + assert result is expected + + +def test_the_managed_depotdownloader_wins_over_path(tmp_path, monkeypatch): + managed = tmp_path / "depotdownloader" + managed.write_text("") + managed.chmod(0o755) + monkeypatch.setattr(deps, "MANAGED_DEPOTDOWNLOADER", managed) + assert deps.depotdownloader_bin() == str(managed) + + +@pytest.fixture +def run_log(tmp_path, monkeypatch): + monkeypatch.setattr(runlog, "LOG_DIR", tmp_path / "state") + log = runlog.setup(1091500) + yield log + for handler in list(log.logger.handlers): + log.logger.removeHandler(handler) + handler.close() + + +def _download_owned(monkeypatch, run_log, tmp_path, outcomes): + monkeypatch.setattr( + appinfo, "get_dlc_plans", lambda appid, language: {7: {"70"}, 8: {"80"}} + ) + monkeypatch.setattr( + depotdl, "download_dlc", lambda dlc_appid, **kw: outcomes[dlc_appid] + ) + return dlc.download_owned(1, tmp_path, "u", "english", 8, run_log) + + +def test_owned_dlc_is_returned_and_unowned_skipped(monkeypatch, run_log, tmp_path): + assert _download_owned(monkeypatch, run_log, tmp_path, {7: True, 8: None}) == { + "70": "7" + } + + +def test_a_failed_dlc_download_aborts(monkeypatch, run_log, tmp_path): + assert _download_owned(monkeypatch, run_log, tmp_path, {7: False, 8: True}) is None + + +def test_downloaded_dlc_is_required_and_credited_to_its_app(steam, monkeypatch): + library, install_dir = steam + (install_dir / ".DepotDownloader" / "2138330_100389.manifest").write_bytes(b"x") + manifest = library.steamapps / "appmanifest_1771300.acf" + state = recognize.read_app_state(manifest) + # Steam's own progress for a DLC download it had started itself. + state["DlcDownloads"] = { + "2138330": {"BytesDownloaded": "1", "BytesToDownload": "9"} + } + manifest.write_text(vdf.dumps("AppState", state)) + _stub_download(monkeypatch) + _stub_steamcmd(monkeypatch, returncode=8) + monkeypatch.setattr(dlc, "download_owned", lambda *a: {"2138330": "2138330"}) + + assert cli.main(ARGV) == 0 + + state = recognize.read_app_state(manifest) + assert state["InstalledDepots"]["2138330"]["dlcappid"] == "2138330" + assert "dlcappid" not in state["InstalledDepots"]["1771303"] + assert "DlcDownloads" not in state + + +def test_a_failed_dlc_download_leaves_the_manifest_alone(steam, monkeypatch): + library, _ = steam + manifest = library.steamapps / "appmanifest_1771300.acf" + before = manifest.read_text() + _stub_download(monkeypatch) + monkeypatch.setattr(dlc, "download_owned", lambda *a: None) + + assert cli.main(ARGV) == 1 + assert manifest.read_text() == before + + +def test_a_dlc_with_a_malformed_depot_section_is_skipped(monkeypatch): + _app_info(monkeypatch, dlc_blocks={**DLC_BLOCKS, 2441600: {"depots": "scalar"}}) + assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330"}} diff --git a/_installer_import/tests/test_finish_check.py b/_installer_import/tests/test_finish_check.py new file mode 100644 index 0000000..78870ed --- /dev/null +++ b/_installer_import/tests/test_finish_check.py @@ -0,0 +1,159 @@ +from steam_installer.finish_check import check_ready_to_finish + + +def test_check_ready_to_finish_blocks_missing_depot(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771302", "1771303"}, + expected_build_id="23914554", + ) + assert any("1771303" in p for p in problems) + + +def test_check_ready_to_finish_blocks_zero_buildid(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "0", "TargetBuildID": "0"}, + required_depots={"1771303"}, + expected_build_id=None, + ) + assert any("buildid" in p for p in problems) + + +def test_check_ready_to_finish_blocks_build_mismatch(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "11111111"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + assert any("different build" in p for p in problems) + + +def test_check_ready_to_finish_blocks_running_steam(tmp_path, monkeypatch): + """Steam owns the manifest while it runs and rewrites it from its own state.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: [4242]) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + assert any("4242" in p for p in problems) + + +def test_check_ready_to_finish_passes_when_everything_holds(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + + assert ( + check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + == [] + ) + + +def test_check_ready_to_finish_blocks_empty_required_depot_set(tmp_path, monkeypatch): + """An empty required set would make the depot check vacuously true and wave + through a download that fetched nothing.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + (install_dir / ".DepotDownloader").mkdir(parents=True) + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots=set(), + expected_build_id="23914554", + ) + assert any("no depots match" in p for p in problems) + + +def test_check_ready_to_finish_blocks_when_no_game_content(tmp_path, monkeypatch): + """Cache filenames are not content: a dir holding only + .DepotDownloader/_.manifest must not be written as installed.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id="23914554", + ) + assert any("no game content" in p for p in problems) + + +def test_check_ready_to_finish_blocks_unknown_public_build(tmp_path, monkeypatch): + """A None expected build means we cannot tell current from stale -- refuse + rather than silently skipping the comparison.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "23914554"}, + required_depots={"1771303"}, + expected_build_id=None, + ) + assert any("current public build" in p for p in problems) + + +def test_gate_blocks_before_staging_is_destroyed(tmp_path, monkeypatch): + """finish_manifest_manually raises when every cached depot is shared. The + CLI deletes Steam's staging tree first, so the gate must catch this while + refusing is still free -- otherwise a failed run destroys tens of GB of + resumable download state.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1001_555.manifest").write_bytes(b"x") + (install_dir / "game.pak").write_bytes(b"x" * 1000) + + problems = check_ready_to_finish( + install_dir=install_dir, + app_state={"buildid": "999", "SharedDepots": {"1001": "228980"}}, + required_depots={"1001"}, + expected_build_id="999", + ) + assert any("nothing to record as installed" in p for p in problems) diff --git a/_installer_import/tests/test_manifest_finish.py b/_installer_import/tests/test_manifest_finish.py new file mode 100644 index 0000000..75dea03 --- /dev/null +++ b/_installer_import/tests/test_manifest_finish.py @@ -0,0 +1,187 @@ +import pytest +from _recognize_helpers import _write_partial_manifest +from steam_installer import vdf +from steam_installer.library import SteamLibrary +from steam_installer.manifest_write import ( + finish_manifest_manually, +) +from steam_installer.recognize import ( + RecognizeError, + download_counters_settled, +) + + +def test_finish_manifest_manually_requires_existing_manifest(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + install_dir = tmp_path / "Steam" / "steamapps" / "common" / "Game" + install_dir.mkdir(parents=True) + with pytest.raises(RecognizeError, match="run steamcmd"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + +def test_finish_manifest_manually_requires_cached_depot_manifests(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + install_dir.mkdir(parents=True) + with pytest.raises(RecognizeError, match="No cached depot manifests"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + +def test_finish_manifest_manually_sets_state_flags_and_depots(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + (install_dir / "game.bin").write_bytes(b"0123456789") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + result = vdf.parse(manifest_path.read_text()) + app_state = result["AppState"] + assert app_state["StateFlags"] == "4" + assert app_state["buildid"] == "23914554" + assert app_state["InstalledDepots"]["1771302"]["manifest"] == "257127247679847518" + assert app_state["SharedDepots"] == {"228989": "228980"} + assert "StagedDepots" not in app_state + assert manifest_path.with_suffix(".acf.bak").is_file() + + +def test_finish_manifest_manually_falls_back_to_target_buildid_when_zero(tmp_path): + """buildid "0" is a sentinel for "no real build installed yet", not a real + value -- a naive `or` chain treats it as truthy and never falls through.""" + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + data = { + "appid": "1771300", + "StateFlags": "1026", + "buildid": "0", + "TargetBuildID": "23914554", + } + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data) + ) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert app_state["buildid"] == "23914554" + + +def test_finish_manifest_manually_corrects_installdir(tmp_path): + """The exact bug that caused real data loss: a stale installdir field + pointing at a different folder than where content was actually downloaded.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + data = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text()) + data["AppState"]["installdir"] = "Some Stale Wrong Name" + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data["AppState"]) + ) + + install_dir = library.common / "KingdomComeDeliverance2" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert app_state["installdir"] == "KingdomComeDeliverance2" + + +def test_finish_manifest_settles_download_counters(tmp_path): + """The regression that started all this: a manifest claiming StateFlags=4 + while still saying 22 GB of 92 GB downloaded. Steam discarded the "installed" + claim and resumed a 90 GB download.""" + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + data = { + "appid": "1771300", + "StateFlags": "1026", + "buildid": "23914554", + "DownloadType": "1", + "BytesToDownload": "92034161488", + "BytesDownloaded": "22129058224", + "BytesToStage": "96447764586", + "BytesStaged": "22462334559", + "StagingSize": "25876914424", + } + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data) + ) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert app_state["BytesDownloaded"] == app_state["BytesToDownload"] + assert app_state["BytesStaged"] == app_state["BytesToStage"] + assert app_state["StagingSize"] == "0" + # DownloadType is present on every real installed manifest -- keep it. + assert app_state["DownloadType"] == "1" + assert download_counters_settled(app_state) + + +def test_finish_manifest_excludes_shared_depots_from_installed(tmp_path): + """228989 is a SharedDepots entry owned by app 228980; claiming this app + installed its own copy contradicts SharedDepots.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "228989_5753583882400741046.manifest").write_bytes(b"x") + (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + manifest_path = finish_manifest_manually( + library, 1771300, install_dir, now_unix=1000 + ) + + app_state = vdf.parse(manifest_path.read_text())["AppState"] + assert set(app_state["InstalledDepots"]) == {"1771303"} + assert app_state["SharedDepots"] == {"228989": "228980"} + + +def test_finish_manifest_rejects_only_shared_depots(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / "228989_5753583882400741046.manifest").write_bytes(b"x") + + with pytest.raises(RecognizeError, match="shared depot"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + +def test_finish_manifest_rejects_malformed_appstate(tmp_path, monkeypatch): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + install_dir = library.common / "Game" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1.manifest").write_bytes(b"x") + monkeypatch.setattr( + "steam_installer.vdf.parse", lambda text: {"AppState": "scalar"} + ) + + with pytest.raises(RecognizeError, match="not a valid AppState"): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1) diff --git a/_installer_import/tests/test_manifest_write.py b/_installer_import/tests/test_manifest_write.py new file mode 100644 index 0000000..0be9ad3 --- /dev/null +++ b/_installer_import/tests/test_manifest_write.py @@ -0,0 +1,157 @@ +import pytest +from _recognize_helpers import _write_partial_manifest +from steam_installer import manifest_write, vdf +from steam_installer.library import SteamLibrary +from steam_installer.manifest_write import ( + ensure_installdir_correct, + finish_manifest_manually, + measure_content_size, +) + + +def test_ensure_installdir_correct_patches_stale_field(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + data = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text()) + data["AppState"]["installdir"] = "Kingdom Come: Deliverance II" + (library.steamapps / "appmanifest_1771300.acf").write_text( + vdf.dumps("AppState", data["AppState"]) + ) + + install_dir = library.common / "KingdomComeDeliverance2" + ensure_installdir_correct(library, 1771300, install_dir) + + app_state = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text())[ + "AppState" + ] + assert app_state["installdir"] == "KingdomComeDeliverance2" + + +def test_ensure_installdir_correct_noop_when_already_correct(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + manifest_path = library.steamapps / "appmanifest_1771300.acf" + data = vdf.parse(manifest_path.read_text()) + data["AppState"]["installdir"] = "KingdomComeDeliverance2" + manifest_path.write_text(vdf.dumps("AppState", data["AppState"])) + before = manifest_path.read_text() + + ensure_installdir_correct( + library, 1771300, library.common / "KingdomComeDeliverance2" + ) + + assert manifest_path.read_text() == before + + +def test_ensure_installdir_correct_noop_when_no_manifest(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + # Should not raise, and should not create a manifest out of nothing. + ensure_installdir_correct(library, 1771300, library.common / "Game") + assert not (library.steamapps / "appmanifest_1771300.acf").exists() + + +def test_measure_content_size_excludes_depotdownloader_cache(tmp_path): + install_dir = tmp_path / "Game" + cache_dir = install_dir / ".DepotDownloader" / "staging" / "Data" + cache_dir.mkdir(parents=True) + (install_dir / "game.bin").write_bytes(b"x" * 100) + (install_dir / ".DepotDownloader" / "d.manifest").write_bytes(b"y" * 5000) + (cache_dir / "staged.pak").write_bytes(b"z" * 9000) + + assert measure_content_size(install_dir) == 100 + + +def test_backup_keeps_the_pristine_manifest_across_reruns(tmp_path): + """A second run must not overwrite the backup with run 1's patched output -- + that turns the recovery artifact for a 90+ GB install into a copy of the + thing it is supposed to protect against.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + manifest = library.steamapps / "appmanifest_1771300.acf" + pristine = manifest.read_text() + install_dir = library.common / "Game" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + finish_manifest_manually(library, 1771300, install_dir, now_unix=2000) + + assert manifest.with_suffix(".acf.bak").read_text() == pristine + + +def test_manifest_write_is_atomic(tmp_path, monkeypatch): + """A crash mid-write must not leave a truncated, unparseable manifest.""" + library = SteamLibrary(tmp_path / "Steam") + _write_partial_manifest(library.steamapps, 1771300) + manifest = library.steamapps / "appmanifest_1771300.acf" + before = manifest.read_text() + install_dir = library.common / "Game" + cache = install_dir / ".DepotDownloader" + cache.mkdir(parents=True) + (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") + + def explode(self, *args, **kwargs): + raise OSError(28, "No space left on device") + + monkeypatch.setattr("pathlib.Path.write_text", explode) + with pytest.raises(OSError): + finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) + + # Original intact, and no temp file left behind. + assert manifest.read_text() == before + assert list(library.steamapps.glob("*.tmp*")) == [] + + +def test_ensure_installdir_correct_ignores_malformed_manifest(tmp_path, monkeypatch): + """Defensive guard: vdf.parse always yields a dict today, so inject a parser + that does not, to keep the guard honest if vdf ever changes.""" + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.write_text(vdf.dumps("AppState", {"installdir": "Stale"})) + before = manifest.read_text() + monkeypatch.setattr( + "steam_installer.vdf.parse", lambda text: {"AppState": "scalar"} + ) + + ensure_installdir_correct(library, 1771300, library.common / "Game") + + assert manifest.read_text() == before + + +def test_ensure_installdir_correct_is_a_noop_when_already_right(tmp_path): + library = SteamLibrary(tmp_path / "Steam") + library.steamapps.mkdir(parents=True) + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.write_text(vdf.dumps("AppState", {"installdir": "Game"})) + + ensure_installdir_correct(library, 1771300, library.common / "Game") + + assert not manifest.with_suffix(".acf.bak").exists() + + +def test_write_atomic_cleans_up_its_temp_file_on_failure(tmp_path, monkeypatch): + target = tmp_path / "appmanifest_1.acf" + target.write_text("original") + + def explode(self, *args, **kwargs): + raise OSError(28, "No space left on device") + + monkeypatch.setattr("pathlib.Path.write_text", explode) + with pytest.raises(OSError): + manifest_write.write_atomic(target, "new content") + + assert list(tmp_path.glob("*.tmp*")) == [] + + +def test_measure_content_size_skips_directories_and_symlinks(tmp_path): + """Directories have a size of their own, and a symlink would double-count + its target -- neither is game content Steam should see in SizeOnDisk.""" + install_dir = tmp_path / "Game" + (install_dir / "Data" / "Levels").mkdir(parents=True) + (install_dir / "Data" / "real.pak").write_bytes(b"x" * 250) + (install_dir / "Data" / "link.pak").symlink_to(install_dir / "Data" / "real.pak") + + assert measure_content_size(install_dir) == 250 diff --git a/_installer_import/tests/test_recognize.py b/_installer_import/tests/test_recognize.py index 8a9f258..c4a85d3 100644 --- a/_installer_import/tests/test_recognize.py +++ b/_installer_import/tests/test_recognize.py @@ -1,26 +1,18 @@ -import logging -import signal from pathlib import Path -import pytest - -from steam_installer import recognize, vdf +from steam_installer import vdf from steam_installer.library import SteamLibrary from steam_installer.recognize import ( - RecognizeError, - check_ready_to_finish, clear_steam_staging, discover_cached_depot_manifests, download_counters_settled, - ensure_installdir_correct, - finish_manifest_manually, manifest_reflects_real_install, - measure_content_size, read_app_state, - run_steamcmd_validate, steam_client_pids, ) +from _recognize_helpers import _fake_proc + def test_discover_cached_depot_manifests_parses_filenames(tmp_path): cache_dir = tmp_path / ".DepotDownloader" @@ -41,150 +33,6 @@ def test_discover_cached_depot_manifests_missing_dir_returns_empty(tmp_path): assert discover_cached_depot_manifests(tmp_path) == {} -def _write_partial_manifest(steamapps: Path, appid: int) -> None: - steamapps.mkdir(parents=True, exist_ok=True) - data = { - "appid": str(appid), - "StateFlags": "1026", - "buildid": "23914554", - "SharedDepots": {"228989": "228980"}, - } - (steamapps / f"appmanifest_{appid}.acf").write_text(vdf.dumps("AppState", data)) - - -def test_finish_manifest_manually_requires_existing_manifest(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - install_dir = tmp_path / "Steam" / "steamapps" / "common" / "Game" - install_dir.mkdir(parents=True) - with pytest.raises(RecognizeError, match="run steamcmd"): - finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) - - -def test_finish_manifest_manually_requires_cached_depot_manifests(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - install_dir = library.common / "Game" - install_dir.mkdir(parents=True) - with pytest.raises(RecognizeError, match="No cached depot manifests"): - finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) - - -def test_finish_manifest_manually_sets_state_flags_and_depots(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - install_dir = library.common / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") - (install_dir / "game.bin").write_bytes(b"0123456789") - - manifest_path = finish_manifest_manually( - library, 1771300, install_dir, now_unix=1000 - ) - - result = vdf.parse(manifest_path.read_text()) - app_state = result["AppState"] - assert app_state["StateFlags"] == "4" - assert app_state["buildid"] == "23914554" - assert app_state["InstalledDepots"]["1771302"]["manifest"] == "257127247679847518" - assert app_state["SharedDepots"] == {"228989": "228980"} - assert "StagedDepots" not in app_state - assert manifest_path.with_suffix(".acf.bak").is_file() - - -def test_finish_manifest_manually_falls_back_to_target_buildid_when_zero(tmp_path): - """buildid "0" is a sentinel for "no real build installed yet", not a real - value -- a naive `or` chain treats it as truthy and never falls through.""" - library = SteamLibrary(tmp_path / "Steam") - library.steamapps.mkdir(parents=True) - data = { - "appid": "1771300", - "StateFlags": "1026", - "buildid": "0", - "TargetBuildID": "23914554", - } - (library.steamapps / "appmanifest_1771300.acf").write_text( - vdf.dumps("AppState", data) - ) - install_dir = library.common / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") - - manifest_path = finish_manifest_manually( - library, 1771300, install_dir, now_unix=1000 - ) - - app_state = vdf.parse(manifest_path.read_text())["AppState"] - assert app_state["buildid"] == "23914554" - - -def test_finish_manifest_manually_corrects_installdir(tmp_path): - """The exact bug that caused real data loss: a stale installdir field - pointing at a different folder than where content was actually downloaded.""" - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - data = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text()) - data["AppState"]["installdir"] = "Some Stale Wrong Name" - (library.steamapps / "appmanifest_1771300.acf").write_text( - vdf.dumps("AppState", data["AppState"]) - ) - - install_dir = library.common / "KingdomComeDeliverance2" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") - - manifest_path = finish_manifest_manually( - library, 1771300, install_dir, now_unix=1000 - ) - - app_state = vdf.parse(manifest_path.read_text())["AppState"] - assert app_state["installdir"] == "KingdomComeDeliverance2" - - -def test_ensure_installdir_correct_patches_stale_field(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - data = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text()) - data["AppState"]["installdir"] = "Kingdom Come: Deliverance II" - (library.steamapps / "appmanifest_1771300.acf").write_text( - vdf.dumps("AppState", data["AppState"]) - ) - - install_dir = library.common / "KingdomComeDeliverance2" - ensure_installdir_correct(library, 1771300, install_dir) - - app_state = vdf.parse((library.steamapps / "appmanifest_1771300.acf").read_text())[ - "AppState" - ] - assert app_state["installdir"] == "KingdomComeDeliverance2" - - -def test_ensure_installdir_correct_noop_when_already_correct(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - manifest_path = library.steamapps / "appmanifest_1771300.acf" - data = vdf.parse(manifest_path.read_text()) - data["AppState"]["installdir"] = "KingdomComeDeliverance2" - manifest_path.write_text(vdf.dumps("AppState", data["AppState"])) - before = manifest_path.read_text() - - ensure_installdir_correct( - library, 1771300, library.common / "KingdomComeDeliverance2" - ) - - assert manifest_path.read_text() == before - - -def test_ensure_installdir_correct_noop_when_no_manifest(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - library.steamapps.mkdir(parents=True) - # Should not raise, and should not create a manifest out of nothing. - ensure_installdir_correct(library, 1771300, library.common / "Game") - assert not (library.steamapps / "appmanifest_1771300.acf").exists() - - def test_manifest_reflects_real_install_true_when_consistent(tmp_path): library = SteamLibrary(tmp_path / "Steam") library.steamapps.mkdir(parents=True) @@ -225,87 +73,6 @@ def test_manifest_reflects_real_install_false_when_no_manifest(tmp_path): assert not manifest_reflects_real_install(library, 1771300, library.common / "Game") -def test_finish_manifest_settles_download_counters(tmp_path): - """The regression that started all this: a manifest claiming StateFlags=4 - while still saying 22 GB of 92 GB downloaded. Steam discarded the "installed" - claim and resumed a 90 GB download.""" - library = SteamLibrary(tmp_path / "Steam") - library.steamapps.mkdir(parents=True) - data = { - "appid": "1771300", - "StateFlags": "1026", - "buildid": "23914554", - "DownloadType": "1", - "BytesToDownload": "92034161488", - "BytesDownloaded": "22129058224", - "BytesToStage": "96447764586", - "BytesStaged": "22462334559", - "StagingSize": "25876914424", - } - (library.steamapps / "appmanifest_1771300.acf").write_text( - vdf.dumps("AppState", data) - ) - install_dir = library.common / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - manifest_path = finish_manifest_manually( - library, 1771300, install_dir, now_unix=1000 - ) - - app_state = vdf.parse(manifest_path.read_text())["AppState"] - assert app_state["BytesDownloaded"] == app_state["BytesToDownload"] - assert app_state["BytesStaged"] == app_state["BytesToStage"] - assert app_state["StagingSize"] == "0" - # DownloadType is present on every real installed manifest -- keep it. - assert app_state["DownloadType"] == "1" - assert download_counters_settled(app_state) - - -def test_finish_manifest_excludes_shared_depots_from_installed(tmp_path): - """228989 is a SharedDepots entry owned by app 228980; claiming this app - installed its own copy contradicts SharedDepots.""" - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - install_dir = library.common / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "228989_5753583882400741046.manifest").write_bytes(b"x") - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - manifest_path = finish_manifest_manually( - library, 1771300, install_dir, now_unix=1000 - ) - - app_state = vdf.parse(manifest_path.read_text())["AppState"] - assert set(app_state["InstalledDepots"]) == {"1771303"} - assert app_state["SharedDepots"] == {"228989": "228980"} - - -def test_finish_manifest_rejects_only_shared_depots(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - install_dir = library.common / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "228989_5753583882400741046.manifest").write_bytes(b"x") - - with pytest.raises(RecognizeError, match="shared depot"): - finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) - - -def test_measure_content_size_excludes_depotdownloader_cache(tmp_path): - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" / "staging" / "Data" - cache_dir.mkdir(parents=True) - (install_dir / "game.bin").write_bytes(b"x" * 100) - (install_dir / ".DepotDownloader" / "d.manifest").write_bytes(b"y" * 5000) - (cache_dir / "staged.pak").write_bytes(b"z" * 9000) - - assert measure_content_size(install_dir) == 100 - - def test_download_counters_settled_detects_partial_download(): assert not download_counters_settled( {"BytesToDownload": "92", "BytesDownloaded": "22"} @@ -335,90 +102,6 @@ def test_manifest_reflects_real_install_false_when_counters_unsettled(tmp_path): assert not manifest_reflects_real_install(library, 1771300, library.common / "Game") -def test_check_ready_to_finish_blocks_missing_depot(tmp_path, monkeypatch): - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771302_257127247679847518.manifest").write_bytes(b"x") - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "23914554"}, - required_depots={"1771302", "1771303"}, - expected_build_id="23914554", - ) - assert any("1771303" in p for p in problems) - - -def test_check_ready_to_finish_blocks_zero_buildid(tmp_path, monkeypatch): - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "0", "TargetBuildID": "0"}, - required_depots={"1771303"}, - expected_build_id=None, - ) - assert any("buildid" in p for p in problems) - - -def test_check_ready_to_finish_blocks_build_mismatch(tmp_path, monkeypatch): - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "11111111"}, - required_depots={"1771303"}, - expected_build_id="23914554", - ) - assert any("different build" in p for p in problems) - - -def test_check_ready_to_finish_blocks_running_steam(tmp_path, monkeypatch): - """Steam owns the manifest while it runs and rewrites it from its own state.""" - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: [4242]) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "23914554"}, - required_depots={"1771303"}, - expected_build_id="23914554", - ) - assert any("4242" in p for p in problems) - - -def test_check_ready_to_finish_passes_when_everything_holds(tmp_path, monkeypatch): - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - (install_dir / "game.pak").write_bytes(b"x" * 1000) - - assert ( - check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "23914554"}, - required_depots={"1771303"}, - expected_build_id="23914554", - ) - == [] - ) - - def test_read_app_state_missing_file(tmp_path): assert read_app_state(tmp_path / "nope.acf") == {} @@ -454,137 +137,6 @@ def test_clear_steam_staging_when_nothing_staged(tmp_path): assert clear_steam_staging(library, 1771300) == 0 -def test_check_ready_to_finish_blocks_empty_required_depot_set(tmp_path, monkeypatch): - """An empty required set would make the depot check vacuously true and wave - through a download that fetched nothing.""" - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - (install_dir / ".DepotDownloader").mkdir(parents=True) - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "23914554"}, - required_depots=set(), - expected_build_id="23914554", - ) - assert any("no depots match" in p for p in problems) - - -def test_backup_keeps_the_pristine_manifest_across_reruns(tmp_path): - """A second run must not overwrite the backup with run 1's patched output -- - that turns the recovery artifact for a 90+ GB install into a copy of the - thing it is supposed to protect against.""" - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - manifest = library.steamapps / "appmanifest_1771300.acf" - pristine = manifest.read_text() - install_dir = library.common / "Game" - cache = install_dir / ".DepotDownloader" - cache.mkdir(parents=True) - (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) - finish_manifest_manually(library, 1771300, install_dir, now_unix=2000) - - assert manifest.with_suffix(".acf.bak").read_text() == pristine - - -def test_manifest_write_is_atomic(tmp_path, monkeypatch): - """A crash mid-write must not leave a truncated, unparseable manifest.""" - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - manifest = library.steamapps / "appmanifest_1771300.acf" - before = manifest.read_text() - install_dir = library.common / "Game" - cache = install_dir / ".DepotDownloader" - cache.mkdir(parents=True) - (cache / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - def explode(self, *args, **kwargs): - raise OSError(28, "No space left on device") - - monkeypatch.setattr("pathlib.Path.write_text", explode) - with pytest.raises(OSError): - finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) - - # Original intact, and no temp file left behind. - assert manifest.read_text() == before - assert list(library.steamapps.glob("*.tmp*")) == [] - - -def test_check_ready_to_finish_blocks_when_no_game_content(tmp_path, monkeypatch): - """Cache filenames are not content: a dir holding only - .DepotDownloader/_.manifest must not be written as installed.""" - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "23914554"}, - required_depots={"1771303"}, - expected_build_id="23914554", - ) - assert any("no game content" in p for p in problems) - - -def test_check_ready_to_finish_blocks_unknown_public_build(tmp_path, monkeypatch): - """A None expected build means we cannot tell current from stale -- refuse - rather than silently skipping the comparison.""" - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1771303_1999516201078544527.manifest").write_bytes(b"x") - (install_dir / "game.pak").write_bytes(b"x" * 1000) - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "23914554"}, - required_depots={"1771303"}, - expected_build_id=None, - ) - assert any("current public build" in p for p in problems) - - -def test_gate_blocks_before_staging_is_destroyed(tmp_path, monkeypatch): - """finish_manifest_manually raises when every cached depot is shared. The - CLI deletes Steam's staging tree first, so the gate must catch this while - refusing is still free -- otherwise a failed run destroys tens of GB of - resumable download state.""" - monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) - install_dir = tmp_path / "Game" - cache_dir = install_dir / ".DepotDownloader" - cache_dir.mkdir(parents=True) - (cache_dir / "1001_555.manifest").write_bytes(b"x") - (install_dir / "game.pak").write_bytes(b"x" * 1000) - - problems = check_ready_to_finish( - install_dir=install_dir, - app_state={"buildid": "999", "SharedDepots": {"1001": "228980"}}, - required_depots={"1001"}, - expected_build_id="999", - ) - assert any("nothing to record as installed" in p for p in problems) - - -def _fake_proc(tmp_path, entries): - """Build a /proc-shaped tree: {pid: exe_target_path_or_None}.""" - proc = tmp_path / "proc" - proc.mkdir() - for name, target in entries.items(): - entry = proc / name - entry.mkdir() - if target is not None: - real = tmp_path / "bin" / target - real.parent.mkdir(parents=True, exist_ok=True) - real.write_text("") - (entry / "exe").symlink_to(real) - return proc - - def test_steam_client_pids_finds_native_and_flatpak(tmp_path, monkeypatch): """Matching on the executable name only: an earlier version also demanded 'ubuntu12_32' in the path, so a Flatpak Steam (/app/bin/steam) silently @@ -618,75 +170,6 @@ def test_steam_client_pids_ignores_vanished_processes(tmp_path, monkeypatch): assert steam_client_pids() == [111] -def test_run_steamcmd_validate_argv_and_result(monkeypatch): - seen = {} - - def fake_run_logged(cmd, logger, timeout=None): - seen["cmd"] = cmd - seen["timeout"] = timeout - return 0, "all good" - - monkeypatch.setattr("steam_installer.recognize.runlog.run_logged", fake_run_logged) - - result = run_steamcmd_validate(1771300, "krzys2000", logging.getLogger("t")) - - cmd = seen["cmd"] - # The platform flag must precede +login, and no force_install_dir may appear: - # it sets a LIBRARY ROOT and pointing it at the game dir caused the original - # "Invalid platform" failure. - assert cmd.index("+@sSteamCmdForcePlatformType") < cmd.index("+login") - assert "force_install_dir" not in " ".join(cmd) - assert cmd[cmd.index("+app_update") + 1 : cmd.index("+app_update") + 3] == [ - "1771300", - "validate", - ] - assert result.completed and result.returncode == 0 - assert result.output == "all good" - assert not result.crashed_on_install_script - - -def test_run_steamcmd_validate_detects_the_install_script_segfault(monkeypatch): - """The known steamcmd bug kills the process with SIGSEGV; checking the - signal beats grepping captured text for a message string.""" - monkeypatch.setattr( - "steam_installer.recognize.runlog.run_logged", - lambda cmd, logger, timeout=None: (-signal.SIGSEGV, ""), - ) - - result = run_steamcmd_validate(1, "u", logging.getLogger("t")) - - assert result.crashed_on_install_script - assert not result.completed - - -def test_ensure_installdir_correct_ignores_malformed_manifest(tmp_path, monkeypatch): - """Defensive guard: vdf.parse always yields a dict today, so inject a parser - that does not, to keep the guard honest if vdf ever changes.""" - library = SteamLibrary(tmp_path / "Steam") - library.steamapps.mkdir(parents=True) - manifest = library.steamapps / "appmanifest_1771300.acf" - manifest.write_text(vdf.dumps("AppState", {"installdir": "Stale"})) - before = manifest.read_text() - monkeypatch.setattr( - "steam_installer.recognize.vdf.parse", lambda text: {"AppState": "scalar"} - ) - - ensure_installdir_correct(library, 1771300, library.common / "Game") - - assert manifest.read_text() == before - - -def test_ensure_installdir_correct_is_a_noop_when_already_right(tmp_path): - library = SteamLibrary(tmp_path / "Steam") - library.steamapps.mkdir(parents=True) - manifest = library.steamapps / "appmanifest_1771300.acf" - manifest.write_text(vdf.dumps("AppState", {"installdir": "Game"})) - - ensure_installdir_correct(library, 1771300, library.common / "Game") - - assert not manifest.with_suffix(".acf.bak").exists() - - def test_manifest_reflects_real_install_false_on_malformed_appstate( tmp_path, monkeypatch ): @@ -696,26 +179,11 @@ def test_manifest_reflects_real_install_false_on_malformed_appstate( vdf.dumps("AppState", {"StateFlags": "4"}) ) monkeypatch.setattr( - "steam_installer.recognize.vdf.parse", lambda text: {"AppState": "scalar"} + "steam_installer.vdf.parse", lambda text: {"AppState": "scalar"} ) assert not manifest_reflects_real_install(library, 1771300, library.common / "Game") -def test_finish_manifest_rejects_malformed_appstate(tmp_path, monkeypatch): - library = SteamLibrary(tmp_path / "Steam") - _write_partial_manifest(library.steamapps, 1771300) - install_dir = library.common / "Game" - cache = install_dir / ".DepotDownloader" - cache.mkdir(parents=True) - (cache / "1771303_1.manifest").write_bytes(b"x") - monkeypatch.setattr( - "steam_installer.recognize.vdf.parse", lambda text: {"AppState": "scalar"} - ) - - with pytest.raises(RecognizeError, match="not a valid AppState"): - finish_manifest_manually(library, 1771300, install_dir, now_unix=1) - - def test_steam_client_pids_skips_unreadable_exe(tmp_path, monkeypatch): """/proc//exe raises for processes we don't own; that must not abort the scan and silently disable the guard.""" @@ -738,28 +206,3 @@ def test_steam_client_pids_skips_unreadable_exe(tmp_path, monkeypatch): monkeypatch.setattr(Path, "resolve", flaky) assert steam_client_pids() == [222] - - -def test_write_atomic_cleans_up_its_temp_file_on_failure(tmp_path, monkeypatch): - target = tmp_path / "appmanifest_1.acf" - target.write_text("original") - - def explode(self, *args, **kwargs): - raise OSError(28, "No space left on device") - - monkeypatch.setattr("pathlib.Path.write_text", explode) - with pytest.raises(OSError): - recognize._write_atomic(target, "new content") - - assert list(tmp_path.glob("*.tmp*")) == [] - - -def test_measure_content_size_skips_directories_and_symlinks(tmp_path): - """Directories have a size of their own, and a symlink would double-count - its target -- neither is game content Steam should see in SizeOnDisk.""" - install_dir = tmp_path / "Game" - (install_dir / "Data" / "Levels").mkdir(parents=True) - (install_dir / "Data" / "real.pak").write_bytes(b"x" * 250) - (install_dir / "Data" / "link.pak").symlink_to(install_dir / "Data" / "real.pak") - - assert measure_content_size(install_dir) == 250 diff --git a/_installer_import/tests/test_steamcmd_validate.py b/_installer_import/tests/test_steamcmd_validate.py new file mode 100644 index 0000000..3a1c474 --- /dev/null +++ b/_installer_import/tests/test_steamcmd_validate.py @@ -0,0 +1,47 @@ +import logging +import signal + +from steam_installer.steamcmd_validate import run_steamcmd_validate + + +def test_run_steamcmd_validate_argv_and_result(monkeypatch): + seen = {} + + def fake_run_logged(cmd, logger, timeout=None): + seen["cmd"] = cmd + seen["timeout"] = timeout + return 0, "all good" + + monkeypatch.setattr( + "steam_installer.steamcmd_validate.runlog.run_logged", fake_run_logged + ) + + result = run_steamcmd_validate(1771300, "krzys2000", logging.getLogger("t")) + + cmd = seen["cmd"] + # The platform flag must precede +login, and no force_install_dir may appear: + # it sets a LIBRARY ROOT and pointing it at the game dir caused the original + # "Invalid platform" failure. + assert cmd.index("+@sSteamCmdForcePlatformType") < cmd.index("+login") + assert "force_install_dir" not in " ".join(cmd) + assert cmd[cmd.index("+app_update") + 1 : cmd.index("+app_update") + 3] == [ + "1771300", + "validate", + ] + assert result.completed and result.returncode == 0 + assert result.output == "all good" + assert not result.crashed_on_install_script + + +def test_run_steamcmd_validate_detects_the_install_script_segfault(monkeypatch): + """The known steamcmd bug kills the process with SIGSEGV; checking the + signal beats grepping captured text for a message string.""" + monkeypatch.setattr( + "steam_installer.steamcmd_validate.runlog.run_logged", + lambda cmd, logger, timeout=None: (-signal.SIGSEGV, ""), + ) + + result = run_steamcmd_validate(1, "u", logging.getLogger("t")) + + assert result.crashed_on_install_script + assert not result.completed From 13510d3a0dacf2194c88ec1100adebbb5daf9d7e Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 4 Oct 2026 16:45:43 +0200 Subject: [PATCH 12/24] repo contract: CLAUDE.md commands, test_changed, CI gate, .gitignore update Co-Authored-By: Claude Opus 5 (1M context) --- .../.github/workflows/repo-contract.yml | 25 ++++++++++ _installer_import/.gitignore | 1 + _installer_import/.pre-commit-config.yaml | 7 +++ _installer_import/CLAUDE.md | 8 +++ .../scripts/check_repo_contract.sh | 26 ++++++++++ _installer_import/scripts/test_changed.sh | 50 +++++++++++++++++++ 6 files changed, 117 insertions(+) create mode 100644 _installer_import/.github/workflows/repo-contract.yml create mode 100644 _installer_import/CLAUDE.md create mode 100755 _installer_import/scripts/check_repo_contract.sh create mode 100755 _installer_import/scripts/test_changed.sh diff --git a/_installer_import/.github/workflows/repo-contract.yml b/_installer_import/.github/workflows/repo-contract.yml new file mode 100644 index 0000000..89d5597 --- /dev/null +++ b/_installer_import/.github/workflows/repo-contract.yml @@ -0,0 +1,25 @@ +name: repo-contract + +# The local pre-commit hook can be skipped (a fresh clone has no hooks), so the +# contract is enforced on push too. Mirrors md-naming.yml. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + repo-contract: + name: repo contract + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check out the shared gate + uses: actions/checkout@v4 + with: + repository: kuhyx/utils + path: .utils + + - name: Enforce the repo contract + run: bash .utils/scripts/check_repo_contract.sh diff --git a/_installer_import/.gitignore b/_installer_import/.gitignore index 472098a..f1efa20 100644 --- a/_installer_import/.gitignore +++ b/_installer_import/.gitignore @@ -5,3 +5,4 @@ __pycache__/ *.egg-info/ .coverage .coverage.* +coverage.xml diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml index ef2f130..5ad7e44 100644 --- a/_installer_import/.pre-commit-config.yaml +++ b/_installer_import/.pre-commit-config.yaml @@ -46,3 +46,10 @@ repos: |Cargo\.(toml|lock)|go\.(mod|sum) |\.fvmrc|\.nvmrc|\.python-version |\.github/workflows/.*\.ya?ml)$ + + - id: repo-contract + name: repo contract (CLAUDE.md Commands + test_changed) + entry: scripts/check_repo_contract.sh + language: system + pass_filenames: false + files: ^(CLAUDE\.md|AGENTS\.md|scripts/test_changed\.sh)$ diff --git a/_installer_import/CLAUDE.md b/_installer_import/CLAUDE.md new file mode 100644 index 0000000..c81e24c --- /dev/null +++ b/_installer_import/CLAUDE.md @@ -0,0 +1,8 @@ +## Commands + +- run: `./run.sh ` +- test: `python3 -m pytest -q` +- test-changed: `scripts/test_changed.sh` +- lint: `pre-commit run --all-files` +- coverage: `python3 -m pytest -q --cov --cov-branch --cov-report=xml` +- coverage-gaps: `coverage-gaps coverage.xml` diff --git a/_installer_import/scripts/check_repo_contract.sh b/_installer_import/scripts/check_repo_contract.sh new file mode 100755 index 0000000..8488858 --- /dev/null +++ b/_installer_import/scripts/check_repo_contract.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# ============================================================================ +# Fail if this repo breaks the shared repo contract. +# +# Thin delegate to the shared gate in ~/src/utils, which owns the contract +# text. Copy this file to scripts/check_repo_contract.sh in a repo; never copy +# the shared gate itself, or the shim would exec itself forever. +# ============================================================================ + +set -euo pipefail + +readonly SHARED_GATE="${UTILS_ROOT:-$HOME/src/utils}/scripts/check_repo_contract.sh" + +main() { + if [[ ! -x "$SHARED_GATE" ]]; then + echo "Error: shared repo-contract gate not found at $SHARED_GATE" >&2 + echo " Clone github.com/kuhyx/utils to ~/src/utils, or set" >&2 + echo " UTILS_ROOT to where it lives." >&2 + exit 1 + fi + + exec bash "$SHARED_GATE" "$@" +} + +main "$@" diff --git a/_installer_import/scripts/test_changed.sh b/_installer_import/scripts/test_changed.sh new file mode 100755 index 0000000..65efb87 --- /dev/null +++ b/_installer_import/scripts/test_changed.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# ============================================================================ +# Run only the tests related to files changed vs HEAD (staged, unstaged and +# untracked). Quiet: failures plus a one-line summary. Falls back to the full +# suite when a change cannot be mapped to tests. Generated by repo_contract. +# ============================================================================ + +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" +CHANGED=() +while IFS= read -r f; do + [[ -n "$f" && -e "$f" ]] && CHANGED+=("$f") +done < <({ git diff --name-only HEAD 2>/dev/null || true; git ls-files --others --exclude-standard; } | sort -u) + +pytest_run() { + if [[ -f uv.lock ]]; then uv run pytest "$@" + elif [[ -f poetry.lock ]]; then poetry run pytest "$@" + else python3 -m pytest "$@"; fi +} + +if grep -qs testmon pyproject.toml requirements*.txt; then + pytest_run --testmon -q --tb=short; exit $? +fi + +full=0 +tests=() +for f in "${CHANGED[@]}"; do + case "$f" in + */conftest.py | conftest.py | pyproject.toml | requirements*.txt | pytest.ini | setup.cfg) full=1 ;; + *.py) + base="$(basename "$f" .py)" + if [[ "$base" == test_* || "$base" == *_test ]]; then + tests+=("$f") + else + while IFS= read -r t; do tests+=("$t"); done \ + < <(git ls-files "*test_${base}.py" "*${base}_test.py") + fi + ;; + esac +done + +if [[ $full -eq 0 && ${#CHANGED[@]} -eq 0 ]]; then echo "no changes vs HEAD: nothing to test"; exit 0; fi +if [[ $full -eq 0 && ${#tests[@]} -gt 0 ]]; then pytest_run -q --tb=short "${tests[@]}"; exit $?; fi +if [[ $full -eq 0 ]] && ! printf '%s\n' "${CHANGED[@]}" | grep -q '\.py$'; then + echo "no python changes: nothing to test"; exit 0 +fi +echo "no mapped tests: running full suite" +pytest_run -q --tb=short From 4d9e845f32ad55a15b71f3fa44f883c402b6197e Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 24 Oct 2026 08:36:40 +0200 Subject: [PATCH 13/24] Finish a completed download: skip placeholder depots, prove the build from gids A 160 GB Baldur's Gate 3 download (2026-10-23) was refused at the finish: - it demanded depots 1419654/5/6, which app info lists with no public manifest -- nothing depotdownloader can fetch. The depot plan now skips depots without a manifest on the public branch. - the enforcer's stub manifest said build 0. Every required depot (own-app DLC included) whose cached manifest matches the public branch's gid now proves the current build, which is what the finished manifest records; a stale gid is refused. The installdir is also corrected as soon as Steam is closed, so a stub named after the display name never sends Steam's resumed download elsewhere. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GYmCezqhUjUmMisF5YtM4F --- _installer_import/steam_installer/appinfo.py | 34 ++++++++-- _installer_import/steam_installer/dlc.py | 13 ++-- _installer_import/steam_installer/finalize.py | 11 ++++ .../steam_installer/finish_check.py | 39 +++++++++++ _installer_import/steam_installer/flow.py | 19 ++++-- .../steam_installer/manifest_write.py | 5 +- _installer_import/tests/test_appinfo.py | 24 ++++--- .../tests/test_appinfo_public_manifests.py | 44 +++++++++++++ _installer_import/tests/test_cli_finish.py | 42 ++++++++++++ _installer_import/tests/test_dlc.py | 64 +++++++++++++++---- _installer_import/tests/test_finish_check.py | 49 ++++++++++++++ 11 files changed, 302 insertions(+), 42 deletions(-) create mode 100644 _installer_import/tests/test_appinfo_public_manifests.py diff --git a/_installer_import/steam_installer/appinfo.py b/_installer_import/steam_installer/appinfo.py index a32c654..236edeb 100644 --- a/_installer_import/steam_installer/appinfo.py +++ b/_installer_import/steam_installer/appinfo.py @@ -9,6 +9,10 @@ from dataclasses import dataclass, field from steam_installer import vdf +# depotdownloader is never given -branch, so it always fetches this one. +PUBLIC_BRANCH = "public" + + class AppInfoError(Exception): pass @@ -34,6 +38,9 @@ class DepotPlan: # depot id -> the DLC app it belongs to. Steam writes that dlcappid into # InstalledDepots, and a DLC depot recorded without it is not credited. dlc: dict[str, str] = field(default_factory=dict) + # depot id -> the manifest gid the public branch ships, for every depot in + # the app's list. A cached depot manifest matching it proves the build. + manifests: dict[str, str] = field(default_factory=dict) @property def all_expected(self) -> set[str]: @@ -147,13 +154,25 @@ def get_depot_plan( for depot_id, entry in depots.items(): if isinstance(entry, dict) and isinstance(entry.get("dlcappid"), str): plan.dlc[depot_id] = entry["dlcappid"] + gid = _public_gid(entry) + if gid is not None: + plan.manifests[depot_id] = gid return plan +def _public_gid(entry: object) -> str | None: + """The manifest gid a depot ships on the public branch, if it ships one.""" + manifests = entry.get("manifests") if isinstance(entry, dict) else None + public = manifests.get(PUBLIC_BRANCH) if isinstance(manifests, dict) else None + gid = public.get("gid") if isinstance(public, dict) else None + return gid if isinstance(gid, str) else None + + def get_dlc_plans( appid: int, language: str = "english", os_name: str = "windows" -) -> dict[int, set[str]]: - """DLC app id -> the depots it installs itself, for DLC with depots of its own. +) -> dict[int, dict[str, str]]: + """DLC app id -> {depot: public gid} it installs itself, for DLC with depots + of its own. Some DLC (Cyberpunk 2077's Phantom Liberty, app 2138330, ~25 GB) keeps its content under its own app id rather than in the game's depot list, so a @@ -168,13 +187,14 @@ def get_dlc_plans( return {} game_depots = app_block.get("depots", {}) game_depot_ids = set(game_depots) if isinstance(game_depots, dict) else set() - plans: dict[int, set[str]] = {} + plans: dict[int, dict[str, str]] = {} for dlc_id, dlc_block in _fetch_app_blocks(dlc_ids).items(): depots = dlc_block.get("depots", {}) if not isinstance(depots, dict): continue - own = _plan_from_depots(dlc_id, depots, language, os_name).required - own -= game_depot_ids + plan = _plan_from_depots(dlc_id, depots, language, os_name) + # Every required depot has a public gid: the plan only takes those. + own = {d: _public_gid(depots[d]) for d in plan.required - game_depot_ids} if own: plans[dlc_id] = own return plans @@ -198,6 +218,10 @@ def _plan_from_depots( dlc_owner = entry.get("dlcappid") if (dlc_owner and dlc_owner != str(appid)) or entry.get("optional") == "1": continue + # A placeholder (BG3 lists 1419654 as `{}`): no manifest on the branch + # depotdownloader fetches means nothing to download, so never missing. + if _public_gid(entry) is None: + continue config = entry.get("config", {}) config = config if isinstance(config, dict) else {} oslist = config.get("oslist") diff --git a/_installer_import/steam_installer/dlc.py b/_installer_import/steam_installer/dlc.py index 11f2c20..bd621f2 100644 --- a/_installer_import/steam_installer/dlc.py +++ b/_installer_import/steam_installer/dlc.py @@ -26,16 +26,17 @@ def download_owned( language: str, max_downloads: int, log: runlog.RunLog, -) -> dict[str, str] | None: +) -> appinfo.DepotPlan | None: """Download every owned DLC of `appid` that has depots of its own. Returns: - depot id -> DLC app id for each DLC depot now downloaded, or None if a - DLC download failed for a reason other than not owning it. + The DLC depots now downloaded -- each credited to its DLC app, with + its public gid -- or None if a DLC download failed for a reason other + than not owning it. """ logger = log.logger plans = appinfo.get_dlc_plans(appid, language) - downloaded: dict[str, str] = {} + downloaded = appinfo.DepotPlan(required=set(), shared=set()) for dlc_appid, depots in sorted(plans.items()): log.section(f"depotdownloader DLC {dlc_appid}") outcome = depotdl.download_dlc( @@ -56,5 +57,7 @@ def download_owned( ) return None logger.info("Downloaded DLC %d (depots %s).", dlc_appid, sorted(depots)) - downloaded.update(dict.fromkeys(depots, str(dlc_appid))) + downloaded.required |= set(depots) + downloaded.dlc.update(dict.fromkeys(depots, str(dlc_appid))) + downloaded.manifests.update(depots) return downloaded diff --git a/_installer_import/steam_installer/finalize.py b/_installer_import/steam_installer/finalize.py index cfff1d1..663ad7a 100644 --- a/_installer_import/steam_installer/finalize.py +++ b/_installer_import/steam_installer/finalize.py @@ -52,6 +52,7 @@ def finish( app_state=app_state, required_depots=depot_plan.required, expected_build_id=expected_build_id, + expected_manifests=depot_plan.manifests, ) if problems: # Fail closed. Writing StateFlags=4 here is what previously turned a @@ -76,6 +77,16 @@ def finish( install_dir=target, now_unix=int(time.time()), dlc_depots=depot_plan.dlc, + # Proven depot by depot, so it outranks whatever the manifest says. + build_id=( + expected_build_id + if finish_check.build_proven( + recognize.discover_cached_depot_manifests(target), + depot_plan.required, + depot_plan.manifests, + ) + else None + ), ) log.file_contents("Manifest after manual finish", written) diff --git a/_installer_import/steam_installer/finish_check.py b/_installer_import/steam_installer/finish_check.py index 50d5694..18e372e 100644 --- a/_installer_import/steam_installer/finish_check.py +++ b/_installer_import/steam_installer/finish_check.py @@ -16,6 +16,7 @@ def check_ready_to_finish( app_state: dict, required_depots: set[str], expected_build_id: str | None, + expected_manifests: dict[str, str] | None = None, ) -> list[str]: """Reasons the manifest must NOT be hand-finished. Empty list means go ahead. @@ -57,6 +58,28 @@ def check_ready_to_finish( f"{install_dir}/{recognize.DEPOTDOWNLOADER_DIR} -- the download did not finish" ) + expected_manifests = expected_manifests or {} + stale = sorted( + depot + for depot in required_depots & set(cached) & set(expected_manifests) + if cached[depot] != expected_manifests[depot] + ) + if stale: + problems.append( + f"depot(s) {', '.join(stale)} hold a manifest the public branch no " + "longer ships -- stale content" + ) + if build_proven(cached, required_depots, expected_manifests): + # The manifest's buildid only ever stood in for this proof. A stub + # (StateFlags 1026, buildid 0) that Steam never got to fill in is no + # reason to refuse content that matches the current build depot by depot. + if expected_build_id is None: + problems.append( + "could not determine the current public build from app info, " + "so there is no buildid to record" + ) + return problems + _content_and_steam_problems(install_dir) + buildid = app_state.get("buildid") target = app_state.get("TargetBuildID") if not (buildid and buildid != "0") and not (target and target != "0"): @@ -78,6 +101,22 @@ def check_ready_to_finish( f"{expected_build_id} -- content is for a different build" ) + return problems + _content_and_steam_problems(install_dir) + + +def build_proven( + cached: dict[str, str], required_depots: set[str], expected: dict[str, str] +) -> bool: + """Whether every required depot holds the manifest the public branch ships.""" + return bool(required_depots) and all( + depot in expected and cached.get(depot) == expected[depot] + for depot in required_depots + ) + + +def _content_and_steam_problems(install_dir: Path) -> list[str]: + """The checks that hold however the build was established.""" + problems: list[str] = [] # Cache *filenames* are not content. Without this, an install dir holding # only `.DepotDownloader/_.manifest` and no game files passes every # other check and gets written as StateFlags=4 with SizeOnDisk=0. diff --git a/_installer_import/steam_installer/flow.py b/_installer_import/steam_installer/flow.py index eab8e8a..3b6fcef 100644 --- a/_installer_import/steam_installer/flow.py +++ b/_installer_import/steam_installer/flow.py @@ -85,6 +85,11 @@ def _download_and_recognize( ) -> int: logger = log.logger + # First thing with Steam closed: whatever happens next, Steam resumes onto + # this field, and a stub written from the display name ("Baldur's Gate 3") + # would point it at a folder the download never fills. + manifest_write.ensure_installdir_correct(library, metadata.appid, install_target) + if staged is not None: log.section("Adopting Steam's staged download") logger.info( @@ -125,16 +130,18 @@ def _download_and_recognize( return 1 logger.info("Checking for owned DLC with depots of its own...") - dlc_depots = dlc.download_owned( + dlc_plan = dlc.download_owned( metadata.appid, install_target, username, language, args.max_downloads, log ) - if dlc_depots is None: + if dlc_plan is None: logger.info("Run log (send this): %s", log.path) return 1 # Downloaded DLC is part of this install now: the finish gate must find its - # depot cached, and the manifest must credit it to its DLC app. - depot_plan.required |= set(dlc_depots) - depot_plan.dlc.update(dlc_depots) + # depot cached at its public gid, and the manifest must credit it to its + # DLC app. + depot_plan.required |= dlc_plan.required + depot_plan.dlc.update(dlc_plan.dlc) + depot_plan.manifests.update(dlc_plan.manifests) log.section("depotdownloader cache") for depot_id, mid in sorted( @@ -142,8 +149,6 @@ def _download_and_recognize( ): logger.debug("depot %s -> manifest %s", depot_id, mid) - manifest_write.ensure_installdir_correct(library, metadata.appid, install_target) - if not args.skip_steamcmd: logger.info("Reconciling with steamcmd (validate)...") log.section("steamcmd app_update validate") diff --git a/_installer_import/steam_installer/manifest_write.py b/_installer_import/steam_installer/manifest_write.py index a746021..307aa15 100644 --- a/_installer_import/steam_installer/manifest_write.py +++ b/_installer_import/steam_installer/manifest_write.py @@ -72,6 +72,7 @@ def finish_manifest_manually( install_dir: Path, now_unix: int, dlc_depots: dict[str, str] | None = None, + build_id: str | None = None, ) -> Path: """Patch appmanifest_.acf to StateFlags=4 using only verified-known values. @@ -81,6 +82,8 @@ def finish_manifest_manually( `dlc_depots` maps a DLC depot to its DLC app id. Steam's own manifests record each DLC depot with that `dlcappid`, so it is written the same way. + `build_id`, when given, is a build the caller proved from the cached depot + manifests; it replaces the one Steam or steamcmd recorded. """ manifest_path = library.steamapps / f"appmanifest_{appid}.acf" if not manifest_path.is_file(): @@ -107,7 +110,7 @@ def finish_manifest_manually( # never falls through to TargetBuildID, leaving a stale/wrong buildid. existing_buildid = app_state.get("buildid") buildid = existing_buildid if existing_buildid and existing_buildid != "0" else None - buildid = buildid or app_state.get("TargetBuildID") or "0" + buildid = build_id or buildid or app_state.get("TargetBuildID") or "0" # A shared depot (e.g. Steamworks Common Redistributables) is installed once # under its owning app and only referenced here. Listing it in diff --git a/_installer_import/tests/test_appinfo.py b/_installer_import/tests/test_appinfo.py index 06ab228..f3317ff 100644 --- a/_installer_import/tests/test_appinfo.py +++ b/_installer_import/tests/test_appinfo.py @@ -7,17 +7,20 @@ def _app_block(monkeypatch, block): monkeypatch.setattr(appinfo, "_fetch_app_block", lambda appid: block) +# Every real depot that has content ships a manifest on the public branch. +_PUB = {"manifests": {"public": {"gid": "1"}}} + # Shape taken from a real `app_info_print 1771300`: 12 depots, of which an # english/windows install only ever fetches three. KCD2_DEPOTS = { - "228989": {"depotfromapp": "228980", "config": {"oslist": "windows"}}, - "1771302": {"config": {"oslist": "windows"}}, - "1771303": {"config": {"oslist": "windows"}}, - "1771304": {"dlcappid": "3406020", "config": {"oslist": "windows"}}, - "1771305": {"config": {"oslist": "windows", "language": "czech"}}, - "1771306": {"config": {"oslist": "windows", "language": "english"}}, - "1771307": {"config": {"oslist": "windows", "language": "french"}}, - "4075290": {"dlcappid": "4075290"}, + "228989": {"depotfromapp": "228980", "config": {"oslist": "windows"}, **_PUB}, + "1771302": {"config": {"oslist": "windows"}, **_PUB}, + "1771303": {"config": {"oslist": "windows"}, **_PUB}, + "1771304": {"dlcappid": "3406020", "config": {"oslist": "windows"}, **_PUB}, + "1771305": {"config": {"oslist": "windows", "language": "czech"}, **_PUB}, + "1771306": {"config": {"oslist": "windows", "language": "english"}, **_PUB}, + "1771307": {"config": {"oslist": "windows", "language": "french"}, **_PUB}, + "4075290": {"dlcappid": "4075290", **_PUB}, "branches": {"public": {"buildid": "23914554"}}, } @@ -42,14 +45,14 @@ def test_get_depot_plan_switches_language(monkeypatch): def test_get_depot_plan_excludes_other_os(monkeypatch): _app_block( monkeypatch, - {"depots": {"10": {"config": {"oslist": "linux"}}, "11": {"config": {}}}}, + {"depots": {"10": {"config": {"oslist": "linux"}, **_PUB}, "11": {"config": {}, **_PUB}}}, ) plan = appinfo.get_depot_plan(1771300, os_name="windows") assert plan.required == {"11"} def test_get_depot_plan_skips_optional_depots(monkeypatch): - _app_block(monkeypatch, {"depots": {"10": {"optional": "1"}, "11": {}}}) + _app_block(monkeypatch, {"depots": {"10": {"optional": "1", **_PUB}, "11": _PUB}}) assert appinfo.get_depot_plan(1771300).required == {"11"} @@ -234,3 +237,4 @@ def test_get_build_id_returns_none_when_buildid_not_a_string(monkeypatch): appinfo.subprocess, "run", lambda cmd, **kw: _FakeCompleted(text) ) assert appinfo.get_build_id(1) is None + diff --git a/_installer_import/tests/test_appinfo_public_manifests.py b/_installer_import/tests/test_appinfo_public_manifests.py new file mode 100644 index 0000000..0e7a1ac --- /dev/null +++ b/_installer_import/tests/test_appinfo_public_manifests.py @@ -0,0 +1,44 @@ +"""Depot plans against what the public branch actually ships. + +Split from test_appinfo.py (250-line cap): these are the depots-without-a- +manifest cases that made a finished BG3 download read as incomplete. +""" + +from steam_installer import appinfo + + +def _app_block(monkeypatch, block): + monkeypatch.setattr(appinfo, "_fetch_app_block", lambda appid: block) + + +# Shape taken from a real `app_info_print 1086940` (Baldur's Gate 3, build +# 25605617): depots listed with no manifest for the public branch are +# placeholders. depotdownloader has nothing to fetch for them, so demanding +# them turned a finished 160 GB download into "the download did not finish". +BG3_DEPOTS = { + "1086941": { + "config": {"oslist": "windows"}, + "manifests": {"public": {"gid": "8253106406752983615"}}, + }, + "1419654": {}, + "1419655": {"config": {"oslist": "windows"}}, + "1419656": {"manifests": {"release_patch_7_hf3": {"gid": "1"}}}, + "2378500": { + "dlcappid": "2378500", + "manifests": {"public": {"gid": "6453229909780803137"}}, + }, +} + + +def test_get_depot_plan_skips_depots_without_a_public_manifest(monkeypatch): + _app_block(monkeypatch, {"depots": BG3_DEPOTS}) + assert appinfo.get_depot_plan(1086940).required == {"1086941"} + + +def test_get_depot_plan_records_every_public_manifest(monkeypatch): + """DLC depots too: the finish check proves the build from these gids.""" + _app_block(monkeypatch, {"depots": BG3_DEPOTS}) + assert appinfo.get_depot_plan(1086940).manifests == { + "1086941": "8253106406752983615", + "2378500": "6453229909780803137", + } diff --git a/_installer_import/tests/test_cli_finish.py b/_installer_import/tests/test_cli_finish.py index 29a2d06..5cf6297 100644 --- a/_installer_import/tests/test_cli_finish.py +++ b/_installer_import/tests/test_cli_finish.py @@ -8,6 +8,7 @@ from steam_installer import ( appinfo, cli, recognize, + vdf, ) from _cli_helpers import ARGV, _stub_download, _stub_steamcmd @@ -116,3 +117,44 @@ def test_module_is_runnable_as_a_script(coverage_subprocess_env): ) assert result.returncode == 0 assert "--skip-steamcmd" in result.stdout + + +def test_a_stub_manifest_is_finished_with_the_proven_build(steam, monkeypatch): + """The enforcer's stub says build 0 and Steam never ran to fill it in. The + cached depot manifest matching the public branch proves the current build, + which is what the finished manifest must record (BG3, 2026-10-23).""" + library, install_dir = steam + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.write_text( + vdf.dumps("AppState", {"appid": "1771300", "StateFlags": "1026", "buildid": "0"}) + ) + (gid,) = recognize.discover_cached_depot_manifests(install_dir).values() + monkeypatch.setattr( + appinfo, + "get_depot_plan", + lambda appid, language: appinfo.DepotPlan( + {"1771303"}, {"228989"}, manifests={"1771303": gid} + ), + ) + _stub_download(monkeypatch) + + assert cli.main([*ARGV, "--skip-steamcmd"]) == 0 + app_state = vdf.parse(manifest.read_text())["AppState"] + assert (app_state["StateFlags"], app_state["buildid"]) == ("4", "23914554") + + +def test_the_install_folder_is_corrected_even_when_the_download_fails( + steam, monkeypatch +): + """The enforcer's stub names the folder after the display name ("Baldur's + Gate 3", not "Baldurs Gate 3"). Steam resumes onto whatever the manifest + says, so it is fixed as soon as Steam is closed, not only on success.""" + library, _ = steam + manifest = library.steamapps / "appmanifest_1771300.acf" + state = recognize.read_app_state(manifest) + state["installdir"] = "Kingdom Come: Deliverance II" + manifest.write_text(vdf.dumps("AppState", state)) + _stub_download(monkeypatch, returncode=2) + + assert cli.main([*ARGV, "--skip-steamcmd"]) == 1 + assert recognize.read_app_state(manifest)["installdir"] == "KingdomComeDeliverance2" diff --git a/_installer_import/tests/test_dlc.py b/_installer_import/tests/test_dlc.py index 2b49351..0ef3652 100644 --- a/_installer_import/tests/test_dlc.py +++ b/_installer_import/tests/test_dlc.py @@ -15,11 +15,14 @@ from steam_installer import appinfo, cli, depotdl, deps, dlc, recognize, runlog, pytestmark = pytest.mark.usefixtures("cli_logs") +# Every real depot that has content ships a manifest on the public branch. +_PUB = {"manifests": {"public": {"gid": "1"}}} + GAME = { "extended": {"listofdlc": "2060310,2138330,2441600"}, "depots": { - "1091501": {"config": {"oslist": "windows"}}, - "2060310": {"dlcappid": "2060310", "config": {"oslist": "windows"}}, + "1091501": {"config": {"oslist": "windows"}, **_PUB}, + "2060310": {"dlcappid": "2060310", "config": {"oslist": "windows"}, **_PUB}, }, } DLC_BLOCKS = { @@ -28,8 +31,8 @@ DLC_BLOCKS = { # Its own depot, tagged with its own id -- exactly what it installs. 2138330: { "depots": { - "2138330": {"dlcappid": "2138330", "config": {"oslist": "windows"}}, - "2138331": {"config": {"oslist": "windows", "language": "german"}}, + "2138330": {"dlcappid": "2138330", "config": {"oslist": "windows"}, **_PUB}, + "2138331": {"config": {"oslist": "windows", "language": "german"}, **_PUB}, } }, 2441600: {"common": {"name": "Pre-Order Bonus"}}, @@ -45,7 +48,7 @@ def _app_info(monkeypatch, game=GAME, dlc_blocks=DLC_BLOCKS): def test_only_dlc_with_depots_of_its_own_needs_a_run(monkeypatch): _app_info(monkeypatch) - assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330"}} + assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330": "1"}} @pytest.mark.parametrize("game", [{}, {"extended": "scalar"}, {"extended": {}}]) @@ -57,7 +60,7 @@ def test_a_game_without_dlc_needs_no_lookup(monkeypatch, game): def test_a_game_with_no_depot_list_still_finds_its_dlc(monkeypatch): _app_info(monkeypatch, game={"extended": {"listofdlc": "2138330"}}) - assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330"}} + assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330": "1"}} def test_depot_plan_tags_the_games_own_dlc_depots(monkeypatch): @@ -128,7 +131,9 @@ def run_log(tmp_path, monkeypatch): def _download_owned(monkeypatch, run_log, tmp_path, outcomes): monkeypatch.setattr( - appinfo, "get_dlc_plans", lambda appid, language: {7: {"70"}, 8: {"80"}} + appinfo, + "get_dlc_plans", + lambda appid, language: {7: {"70": "g70"}, 8: {"80": "g80"}}, ) monkeypatch.setattr( depotdl, "download_dlc", lambda dlc_appid, **kw: outcomes[dlc_appid] @@ -137,9 +142,10 @@ def _download_owned(monkeypatch, run_log, tmp_path, outcomes): def test_owned_dlc_is_returned_and_unowned_skipped(monkeypatch, run_log, tmp_path): - assert _download_owned(monkeypatch, run_log, tmp_path, {7: True, 8: None}) == { - "70": "7" - } + plan = _download_owned(monkeypatch, run_log, tmp_path, {7: True, 8: None}) + assert (plan.required, plan.dlc) == ({"70"}, {"70": "7"}) + # Its public gid too, so the finish check can prove the DLC's build. + assert plan.manifests == {"70": "g70"} def test_a_failed_dlc_download_aborts(monkeypatch, run_log, tmp_path): @@ -158,7 +164,7 @@ def test_downloaded_dlc_is_required_and_credited_to_its_app(steam, monkeypatch): manifest.write_text(vdf.dumps("AppState", state)) _stub_download(monkeypatch) _stub_steamcmd(monkeypatch, returncode=8) - monkeypatch.setattr(dlc, "download_owned", lambda *a: {"2138330": "2138330"}) + monkeypatch.setattr(dlc, "download_owned", lambda *a: _dlc_plan("100389")) assert cli.main(ARGV) == 0 @@ -168,17 +174,47 @@ def test_downloaded_dlc_is_required_and_credited_to_its_app(steam, monkeypatch): assert "DlcDownloads" not in state +def _dlc_plan(gid): + return appinfo.DepotPlan( + {"2138330"}, set(), {"2138330": "2138330"}, {"2138330": gid} + ) + + +def test_a_stub_with_own_app_dlc_finishes_with_the_proven_build(steam, monkeypatch): + """Base game and its own-app DLC both match the public branch: a build-0 + stub is finished, the DLC's gid coming from the DLC's own app info.""" + library, install_dir = steam + (install_dir / ".DepotDownloader" / "2138330_100389.manifest").write_bytes(b"x") + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.write_text(vdf.dumps("AppState", {"appid": "1771300", "buildid": "0"})) + monkeypatch.setattr( + appinfo, + "get_depot_plan", + lambda appid, language: appinfo.DepotPlan( + {"1771303"}, set(), manifests={"1771303": "1999516201078544527"} + ), + ) + _stub_download(monkeypatch) + monkeypatch.setattr(dlc, "download_owned", lambda *a: _dlc_plan("100389")) + + assert cli.main([*ARGV, "--skip-steamcmd"]) == 0 + assert recognize.read_app_state(manifest)["buildid"] == "23914554" + + def test_a_failed_dlc_download_leaves_the_manifest_alone(steam, monkeypatch): library, _ = steam manifest = library.steamapps / "appmanifest_1771300.acf" - before = manifest.read_text() + before = recognize.read_app_state(manifest) _stub_download(monkeypatch) monkeypatch.setattr(dlc, "download_owned", lambda *a: None) assert cli.main(ARGV) == 1 - assert manifest.read_text() == before + # Only the folder name is fixed up front; nothing claims an install. + after = recognize.read_app_state(manifest) + assert after.pop("installdir") == "KingdomComeDeliverance2" + assert after == before def test_a_dlc_with_a_malformed_depot_section_is_skipped(monkeypatch): _app_info(monkeypatch, dlc_blocks={**DLC_BLOCKS, 2441600: {"depots": "scalar"}}) - assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330"}} + assert appinfo.get_dlc_plans(1091500) == {2138330: {"2138330": "1"}} diff --git a/_installer_import/tests/test_finish_check.py b/_installer_import/tests/test_finish_check.py index 78870ed..3fae028 100644 --- a/_installer_import/tests/test_finish_check.py +++ b/_installer_import/tests/test_finish_check.py @@ -157,3 +157,52 @@ def test_gate_blocks_before_staging_is_destroyed(tmp_path, monkeypatch): expected_build_id="999", ) assert any("nothing to record as installed" in p for p in problems) + + +def _bg3_dir(tmp_path, gid="8253106406752983615"): + install_dir = tmp_path / "Baldurs Gate 3" + cache_dir = install_dir / ".DepotDownloader" + cache_dir.mkdir(parents=True) + (cache_dir / f"1086941_{gid}.manifest").write_bytes(b"x") + (install_dir / "Gustav.pak").write_bytes(b"x" * 1000) + return install_dir + + +def test_cached_public_manifests_prove_the_build(tmp_path, monkeypatch): + """The enforcer's stub manifest says build 0, and Steam never ran to fill + in TargetBuildID. Every depot holding the public branch's manifest *is* + the current build, so that is no reason to refuse.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + problems = check_ready_to_finish( + install_dir=_bg3_dir(tmp_path), + app_state={"buildid": "0", "TargetBuildID": "0"}, + required_depots={"1086941"}, + expected_build_id="25605617", + expected_manifests={"1086941": "8253106406752983615"}, + ) + assert problems == [] + + +def test_a_stale_depot_manifest_is_refused(tmp_path, monkeypatch): + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + problems = check_ready_to_finish( + install_dir=_bg3_dir(tmp_path, gid="111"), + app_state={"buildid": "25605617"}, + required_depots={"1086941"}, + expected_build_id="25605617", + expected_manifests={"1086941": "8253106406752983615"}, + ) + assert any("1086941" in p and "stale" in p for p in problems) + + +def test_a_proven_build_still_needs_a_buildid_to_record(tmp_path, monkeypatch): + """Matching gids prove the content, but the manifest needs a number.""" + monkeypatch.setattr("steam_installer.recognize.steam_client_pids", lambda: []) + problems = check_ready_to_finish( + install_dir=_bg3_dir(tmp_path), + app_state={"buildid": "0"}, + required_depots={"1086941"}, + expected_build_id=None, + expected_manifests={"1086941": "8253106406752983615"}, + ) + assert any("no buildid to record" in p for p in problems) From 817447a09cd71a31449db422c80b555eec35476d Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Mon, 2 Nov 2026 13:55:44 +0100 Subject: [PATCH 14/24] Add todo app git hooks Co-Authored-By: Claude Opus 5.5 (1M context) --- _installer_import/.pre-commit-config.yaml | 25 +++++++++++++++++++++++ _installer_import/scripts/todo_hooks.sh | 12 +++++++++++ 2 files changed, 37 insertions(+) create mode 100755 _installer_import/scripts/todo_hooks.sh diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml index 5ad7e44..01e52c5 100644 --- a/_installer_import/.pre-commit-config.yaml +++ b/_installer_import/.pre-commit-config.yaml @@ -3,6 +3,7 @@ # Created when the shared markdown-naming gate was rolled out. The checks # are `language: system` because they are plain bash already on this # machine; pre-commit must not build an isolated env for them. +default_install_hook_types: [pre-commit, post-commit, post-merge, post-rewrite] repos: - repo: local hooks: @@ -16,6 +17,30 @@ repos: language: system files: \.(md|markdown)$ + - id: todo-closer + name: close Todo trailer tasks + entry: scripts/todo_hooks.sh closer + language: system + stages: [post-commit] + always_run: true + pass_filenames: false + + - id: todo-import-pr-merge + name: import PR Todo sections (pull) + entry: scripts/todo_hooks.sh import-pr post-merge + language: system + stages: [post-merge] + always_run: true + pass_filenames: false + + - id: todo-import-pr-rewrite + name: import PR Todo sections (pull --rebase) + entry: scripts/todo_hooks.sh import-pr post-rewrite + language: system + stages: [post-rewrite] + always_run: true + pass_filenames: false + # Shared 250-line cap on every file in the commit, code and prose alike. # Checks only what is staged, so an unrelated commit never goes red, and # it fails rather than warns: no baseline, no allowlist. diff --git a/_installer_import/scripts/todo_hooks.sh b/_installer_import/scripts/todo_hooks.sh new file mode 100755 index 0000000..168dc7b --- /dev/null +++ b/_installer_import/scripts/todo_hooks.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Shim: forward to the shared todo git hooks in ~/src/utils (silent if absent). +set -euo pipefail + +readonly SHARED="${UTILS_ROOT:-$HOME/src/utils}/scripts/todo_hooks.sh" + +main() { + [[ -f "$SHARED" ]] || exit 0 + exec bash "$SHARED" "$@" +} + +main "$@" From 16cb05f3a68a7056b48f66838dafc508ff856a99 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Thu, 19 Nov 2026 17:37:12 +0100 Subject: [PATCH 15/24] =?UTF-8?q?get=20rid=20of=20github=20ci/cd=20version?= =?UTF-8?q?=20check=20of=20package=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Todo: 2556952d --- .../workflows/dependency-freshness.yml | 62 ------------------- _installer_import/.pre-commit-config.yaml | 3 +- .../scripts/check_dependency_freshness.sh | 2 +- 3 files changed, 3 insertions(+), 64 deletions(-) delete mode 100644 _installer_import/.github/workflows/dependency-freshness.yml diff --git a/_installer_import/.github/workflows/dependency-freshness.yml b/_installer_import/.github/workflows/dependency-freshness.yml deleted file mode 100644 index 23a0a73..0000000 --- a/_installer_import/.github/workflows/dependency-freshness.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: dependency freshness - -# Every dependency on its ecosystem's newest STABLE release, enforced on push -# as well as at commit time. The local pre-commit hook only sees the manifests -# in a commit and can be skipped entirely by a fresh clone that never ran -# `pre-commit install`; this runs `--all` over the whole repo. -# -# `--strict` is CI-only. Offline the gate degrades to its cached answer and -# passes, because `--no-verify` is banned locally and a hook that hard-failed -# on a flaky connection would leave no way to commit. A runner is always -# online, so here an undeterminable answer is a real failure. -# -# Expect this to go red for reasons this repo's commit did not cause: an -# upstream release overnight is enough. That is the chosen behaviour -- the -# alternative is a gate that only notices staleness when someone remembers to -# look. - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -jobs: - dependency-freshness: - name: dependencies on newest stable - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - # The gate lives in the shared utils repo so every repo resolves the - # same "latest stable" the same way. Checking it out beats vendoring a - # copy, which is exactly how a gate and its survey start disagreeing. - - name: Check out the shared gate - uses: actions/checkout@v4 - with: - repository: kuhyx/utils - path: .utils - - # "3.x", never a pinned minor: the Python target IS the interpreter the - # gate runs under (registries/toolchain.python_installed), so a runner - # held a release behind this machine reports a repo's `requires-python` - # as excluding "latest" while the same check passes locally. flax-docs-mcp - # hit exactly that -- `>=3.14` green here, red on a 3.13 runner. - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install the gate's two runtime dependencies - run: python -m pip install --upgrade pyyaml packaging - - - name: Cache registry lookups - uses: actions/cache@v4 - with: - path: ~/.cache/dep-freshness - key: dep-freshness-${{ hashFiles('**/pubspec.yaml', '**/pyproject.toml', - '**/requirements*.txt', '**/package.json', '**/Cargo.toml', '**/go.mod') }} - restore-keys: dep-freshness- - - - name: Every dependency is on newest stable - run: bash .utils/scripts/check_dependency_freshness.sh --all --strict diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml index 01e52c5..ce3b75d 100644 --- a/_installer_import/.pre-commit-config.yaml +++ b/_installer_import/.pre-commit-config.yaml @@ -59,7 +59,8 @@ repos: # Runs WITHOUT --strict on purpose. Offline it degrades to the cached # answer and passes with a loud banner; `--no-verify` is banned here, so # a hook that hard-failed on a flaky connection would leave no way to - # commit at all. CI, always online, is the strict authority. + # commit at all. The strict run is local only (CI does not check + # freshness): scripts/check_dependency_freshness.sh --all --strict - id: dependency-freshness name: dependencies on newest stable entry: scripts/check_dependency_freshness.sh diff --git a/_installer_import/scripts/check_dependency_freshness.sh b/_installer_import/scripts/check_dependency_freshness.sh index 7715d3c..49d6e29 100755 --- a/_installer_import/scripts/check_dependency_freshness.sh +++ b/_installer_import/scripts/check_dependency_freshness.sh @@ -11,7 +11,7 @@ # # Usage: # scripts/check_dependency_freshness.sh [ ...] # pre-commit -# scripts/check_dependency_freshness.sh --all --strict # CI +# scripts/check_dependency_freshness.sh --all --strict # full local run # # Exit: 0 current | 1 behind | 2 allowlist wrong | 3 undeterminable (offline) # ============================================================================ From 6b67fcc4f19402ac39174ca01990790f7ec60707 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Thu, 19 Nov 2026 20:47:35 +0100 Subject: [PATCH 16/24] Add scripts/deploy.sh and declare it in CLAUDE.md Co-Authored-By: Claude Opus 5.5 --- _installer_import/CLAUDE.md | 1 + _installer_import/scripts/deploy.sh | 38 +++++++++++ _installer_import/tests/test_deploy.py | 94 ++++++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100755 _installer_import/scripts/deploy.sh create mode 100644 _installer_import/tests/test_deploy.py diff --git a/_installer_import/CLAUDE.md b/_installer_import/CLAUDE.md index c81e24c..6949597 100644 --- a/_installer_import/CLAUDE.md +++ b/_installer_import/CLAUDE.md @@ -6,3 +6,4 @@ - lint: `pre-commit run --all-files` - coverage: `python3 -m pytest -q --cov --cov-branch --cov-report=xml` - coverage-gaps: `coverage-gaps coverage.xml` +- deploy: `scripts/deploy.sh` (fast-forwards kuhy's checkout `~/src/steam-game-installer` to origin/main, refusing a dirty tree, reruns `./run.sh --deps-only`, then checks the CLI starts; idempotent, non-zero unless at origin/main and healthy; test: `tests/test_deploy.py`) diff --git a/_installer_import/scripts/deploy.sh b/_installer_import/scripts/deploy.sh new file mode 100755 index 0000000..a5a19a1 --- /dev/null +++ b/_installer_import/scripts/deploy.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# ============================================================================ +# Deploy origin/main to where this tool runs: kuhy's checkout +# (~/src/steam-game-installer, used through ./run.sh). Refuses a dirty tree, +# fast-forwards it, reruns ./run.sh --deps-only (steamcmd, depotdownloader, +# editable .venv), then starts the CLI as the health check. Nothing is +# published elsewhere: no package, no service, no release. +# Idempotent; non-zero unless the checkout is at origin/main and the CLI starts. +# scripts/deploy.sh +# SGI_DEPLOY_DIR=/path another checkout (the test uses a fixture) +# ============================================================================ + +set -euo pipefail + +readonly DIR="${SGI_DEPLOY_DIR:-$HOME/src/steam-game-installer}" + +die() { + echo "deploy: FAILED ($1)" >&2 + exit 1 +} + +main() { + [[ -d "$DIR/.git" ]] || die "no checkout at $DIR" + [[ -z "$(git -C "$DIR" status --porcelain)" ]] || die "$DIR has local changes" + git -C "$DIR" checkout -q main || die "cannot switch $DIR to main" + git -C "$DIR" pull -q --ff-only origin main || die "cannot fast-forward $DIR" + local head want + head="$(git -C "$DIR" rev-parse HEAD)" + want="$(git -C "$DIR" rev-parse origin/main)" + [[ "$head" == "$want" ]] || die "$DIR is at ${head:0:7}, origin/main is ${want:0:7}" + (cd "$DIR" && ./run.sh --deps-only) || die "dependency install failed" + "$DIR/.venv/bin/python" -m steam_installer.cli --help >/dev/null || + die "health check: the CLI does not start" + echo "deploy: $DIR at ${head:0:7}, deps ok, CLI starts" +} + +main "$@" diff --git a/_installer_import/tests/test_deploy.py b/_installer_import/tests/test_deploy.py new file mode 100644 index 0000000..003210e --- /dev/null +++ b/_installer_import/tests/test_deploy.py @@ -0,0 +1,94 @@ +"""scripts/deploy.sh against a fixture checkout instead of kuhy's real one.""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess + +import pytest + +DEPLOY = Path(__file__).resolve().parent.parent / "scripts" / "deploy.sh" + +# Stubs stand in for the real run.sh / venv so the test installs nothing. +RUN_SH = '#!/bin/bash\n[[ "$1" == --deps-only ]] && touch .deps-ran\n' +VENV_PY = '#!/bin/bash\n[[ -f "$(dirname "$0")/healthy" ]]\n' + + +def _git(cwd: Path, *args: str) -> str: + """Run git quietly in cwd and return stdout.""" + return subprocess.run( + ["git", "-C", str(cwd), *args], check=True, capture_output=True, text=True + ).stdout.strip() + + +def _commit(repo: Path, name: str, body: str, mode: int = 0o644) -> None: + """Write one file into repo and commit it.""" + path = repo / name + path.write_text(body) + path.chmod(mode) + _git(repo, "add", name) + _git(repo, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", name) + + +@pytest.fixture +def checkout(tmp_path: Path) -> tuple[Path, Path]: + """An origin with run.sh and a clone of it that is one commit behind.""" + origin = tmp_path / "origin" + origin.mkdir() + _git(origin, "init", "-q", "-b", "main") + _commit(origin, ".gitignore", ".venv/\n.deps-ran\n") + _commit(origin, "run.sh", RUN_SH, 0o755) + clone = tmp_path / "clone" + subprocess.run(["git", "clone", "-q", str(origin), str(clone)], check=True) + _commit(origin, "new.txt", "new\n") + venv_bin = clone / ".venv" / "bin" + venv_bin.mkdir(parents=True) + (venv_bin / "python").write_text(VENV_PY) + (venv_bin / "python").chmod(0o755) + (venv_bin / "healthy").touch() + return origin, clone + + +def _deploy(clone: Path) -> subprocess.CompletedProcess[str]: + """Run deploy.sh pointed at the fixture clone.""" + env = {**os.environ, "SGI_DEPLOY_DIR": str(clone)} + return subprocess.run( + ["bash", str(DEPLOY)], env=env, capture_output=True, text=True, check=False + ) + + +def test_deploy_fast_forwards_installs_and_checks( + checkout: tuple[Path, Path], +) -> None: + """A clean checkout ends at origin/main with deps run and exit 0.""" + origin, clone = checkout + result = _deploy(clone) + assert result.returncode == 0, result.stderr + assert _git(clone, "rev-parse", "HEAD") == _git(origin, "rev-parse", "HEAD") + assert (clone / ".deps-ran").exists() + assert _deploy(clone).returncode == 0 # idempotent + + +def test_deploy_refuses_dirty_checkout(checkout: tuple[Path, Path]) -> None: + """Local edits in kuhy's checkout are never overwritten.""" + _, clone = checkout + (clone / "run.sh").write_text(RUN_SH + "# local edit\n") + result = _deploy(clone) + assert result.returncode != 0 + assert "local changes" in result.stderr + + +def test_deploy_fails_when_health_check_fails(checkout: tuple[Path, Path]) -> None: + """A CLI that cannot start makes deploy exit non-zero.""" + _, clone = checkout + (clone / ".venv" / "bin" / "healthy").unlink() + result = _deploy(clone) + assert result.returncode != 0 + assert "health" in result.stderr + + +def test_claude_md_declares_deploy() -> None: + """CLAUDE.md names scripts/deploy.sh as the deploy command.""" + claude = DEPLOY.parent.parent / "CLAUDE.md" + assert "- deploy: `scripts/deploy.sh`" in claude.read_text() From ed03bf76e7e0ffacae42bceae0dd076d089a1def Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 21 Nov 2026 08:55:44 +0100 Subject: [PATCH 17/24] Put my logo everywhere Todo: 7808ee0a --- _installer_import/pyproject.toml | 5 ++++- _installer_import/steam_installer/cli.py | 6 +++++- _installer_import/tests/test_cli_credit.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 _installer_import/tests/test_cli_credit.py diff --git a/_installer_import/pyproject.toml b/_installer_import/pyproject.toml index a6609d3..2e02155 100644 --- a/_installer_import/pyproject.toml +++ b/_installer_import/pyproject.toml @@ -3,7 +3,10 @@ name = "steam-game-installer" version = "0.1.0" description = "Fast Steam game installs: depotdownloader for speed, steamcmd for recognition." requires-python = ">=3.14" -dependencies = [] +# kuhy's shared "Created by kuhy" credit, shown at the end of --help. +dependencies = [ + "kuhy-credit @ git+https://github.com/kuhyx/utils@kuhy-credit-v0.3.0#subdirectory=kuhy_credit", +] [project.scripts] steam-game-installer = "steam_installer.cli:main" diff --git a/_installer_import/steam_installer/cli.py b/_installer_import/steam_installer/cli.py index d44b707..27ccd7f 100644 --- a/_installer_import/steam_installer/cli.py +++ b/_installer_import/steam_installer/cli.py @@ -5,6 +5,8 @@ from __future__ import annotations import argparse import sys +from kuhy_credit import credit_line + from steam_installer import ( config, depotdl, @@ -21,7 +23,9 @@ from steam_installer.library import find_steam_root, list_libraries, pick_instal def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description="Download an owned Steam game fast, then make Steam recognize it." + description="Download an owned Steam game fast, then make Steam recognize it.", + # kuhy's shared credit line (kuhy_credit, kuhyx/utils). + epilog=credit_line(), ) parser.add_argument( "target", diff --git a/_installer_import/tests/test_cli_credit.py b/_installer_import/tests/test_cli_credit.py new file mode 100644 index 0000000..f51c455 --- /dev/null +++ b/_installer_import/tests/test_cli_credit.py @@ -0,0 +1,14 @@ +"""``--help`` ends on kuhy's shared credit line (kuhy_credit, kuhyx/utils).""" + +import pytest +from kuhy_credit import URL, credit_line +from steam_installer import cli + + +def test_help_ends_with_the_kuhy_credit(capsys): + with pytest.raises(SystemExit) as exc: + cli.main(["--help"]) + assert exc.value.code == 0 + out = capsys.readouterr().out + assert out.rstrip().endswith(credit_line()) + assert URL in out From e2fc518624f31a2e7341fd97a81743082acc7099 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 22 Nov 2026 14:42:45 +0100 Subject: [PATCH 18/24] Read app info from Steam's cache, drop steamcmd Write the 15-key manifest from scratch when none exists. Co-Authored-By: Claude Opus 5.5 --- _installer_import/steam_installer/appinfo.py | 81 +++++------ .../steam_installer/appinfo_cache.py | 115 ++++++++++++++++ _installer_import/steam_installer/cli.py | 13 +- _installer_import/steam_installer/deps.py | 15 +- _installer_import/steam_installer/finalize.py | 38 ++---- .../steam_installer/finish_check.py | 6 +- _installer_import/steam_installer/flow.py | 46 ------- .../steam_installer/manifest_write.py | 78 ++++++++--- .../steam_installer/steamcmd_validate.py | 79 ----------- _installer_import/tests/_appinfo_blob.py | 49 +++++++ _installer_import/tests/_cli_helpers.py | 14 +- _installer_import/tests/test_appinfo.py | 128 +++++++----------- _installer_import/tests/test_appinfo_cache.py | 81 +++++++++++ _installer_import/tests/test_cli.py | 87 +----------- _installer_import/tests/test_cli_finish.py | 104 +++++++++----- _installer_import/tests/test_deps.py | 15 +- _installer_import/tests/test_dlc.py | 29 +--- _installer_import/tests/test_finish_check.py | 3 + .../tests/test_manifest_finish.py | 2 +- .../tests/test_steamcmd_validate.py | 47 ------- 20 files changed, 501 insertions(+), 529 deletions(-) create mode 100644 _installer_import/steam_installer/appinfo_cache.py delete mode 100644 _installer_import/steam_installer/steamcmd_validate.py create mode 100644 _installer_import/tests/_appinfo_blob.py create mode 100644 _installer_import/tests/test_appinfo_cache.py delete mode 100644 _installer_import/tests/test_steamcmd_validate.py diff --git a/_installer_import/steam_installer/appinfo.py b/_installer_import/steam_installer/appinfo.py index 236edeb..c53e961 100644 --- a/_installer_import/steam_installer/appinfo.py +++ b/_installer_import/steam_installer/appinfo.py @@ -1,12 +1,10 @@ -"""Look up app metadata via steamcmd's app_info_print (anonymous, no login needed).""" +"""App metadata (name, installdir, depots, build) from Steam's local app cache.""" from __future__ import annotations -import re -import subprocess from dataclasses import dataclass, field -from steam_installer import vdf +from steam_installer import appinfo_cache # depotdownloader is never given -branch, so it always fetches this one. @@ -41,56 +39,37 @@ class DepotPlan: # depot id -> the manifest gid the public branch ships, for every depot in # the app's list. A cached depot manifest matching it proves the build. manifests: dict[str, str] = field(default_factory=dict) + # shared depot id -> the app it is installed under (its `depotfromapp`). + # A manifest written from scratch records these as SharedDepots. + shared_owner: dict[str, str] = field(default_factory=dict) @property def all_expected(self) -> set[str]: return self.required | self.shared -def _fetch_app_blocks(appids: list[int]) -> dict[int, dict]: - """App info for every id in `appids`, from a single anonymous steamcmd call. +def _cached_blocks(appids: list[int]) -> dict[int, dict]: + """App info for those of `appids` Steam's local cache holds. - One call for the lot: each steamcmd start costs seconds of login, and a game - can list dozens of DLC. + Read from `appcache/appinfo.vdf` (see :mod:`appinfo_cache`), not from + steamcmd: this tool no longer runs steamcmd at all. """ - prints = [arg for appid in appids for arg in ("+app_info_print", str(appid))] - result = subprocess.run( - [ - "steamcmd", - # Without this, a Linux steamcmd resolves a Windows-only app's depot - # list against the host platform and can come back empty. - "+@sSteamCmdForcePlatformType", - "windows", - "+login", - "anonymous", - *prints, - "+quit", - ], - capture_output=True, - text=True, - # The exit code says nothing useful; a missing block is caught below. - check=False, - timeout=60 + 5 * len(appids), - ) - output = result.stdout - # app_info_print emits `""\n{ ... }` inside a stream of steamcmd's own - # log lines; each block runs from its own header to the next one. - starts = {} - for appid in appids: - match = re.search(rf'^"{appid}"\s*$', output, re.MULTILINE) - if not match: - raise AppInfoError( - f"steamcmd did not return app info for {appid}:\n{output}" - ) - starts[appid] = match.start() - ordered = sorted(starts.values()) + [len(output)] - blocks: dict[int, dict] = {} - for appid, start in starts.items(): - end = ordered[ordered.index(start) + 1] - app_block = vdf.parse(output[start:end]).get(str(appid), {}) - if not isinstance(app_block, dict): - raise AppInfoError(f"unexpected app_info_print structure for {appid}") - blocks[appid] = app_block + try: + apps = appinfo_cache.load() + except appinfo_cache.CacheError as exc: + raise AppInfoError(str(exc)) from exc + return {appid: apps[appid] for appid in appids if appid in apps} + + +def _fetch_app_blocks(appids: list[int]) -> dict[int, dict]: + """App info for every id in `appids`; a missing one is an error with the fix.""" + blocks = _cached_blocks(appids) + missing = [str(appid) for appid in appids if appid not in blocks] + if missing: + raise AppInfoError( + f"app {', '.join(missing)} is not in Steam's local app cache: " + f"{appinfo_cache.FIX}" + ) return blocks @@ -188,7 +167,10 @@ def get_dlc_plans( game_depots = app_block.get("depots", {}) game_depot_ids = set(game_depots) if isinstance(game_depots, dict) else set() plans: dict[int, dict[str, str]] = {} - for dlc_id, dlc_block in _fetch_app_blocks(dlc_ids).items(): + # A DLC the cache lacks is not owned: Steam caches app info for every app + # the account owns (1828 of 7102 listed DLC ids on 2026-11-22), and + # depotdownloader would refuse an unowned one anyway. + for dlc_id, dlc_block in _cached_blocks(dlc_ids).items(): depots = dlc_block.get("depots", {}) if not isinstance(depots, dict): continue @@ -206,12 +188,15 @@ def _plan_from_depots( """The depots of one app's `depots` section that an (os, language) install gets.""" required: set[str] = set() shared: set[str] = set() + shared_owner: dict[str, str] = {} for depot_id, entry in depots.items(): if not depot_id.isdigit() or not isinstance(entry, dict): continue # A shared depot is installed once under its owning app, not here. if "depotfromapp" in entry or "sharedinstall" in entry: shared.add(depot_id) + if isinstance(entry.get("depotfromapp"), str): + shared_owner[depot_id] = entry["depotfromapp"] continue # Another app's DLC is not part of this install -- but a DLC app's own # depots carry its own id as dlcappid, and are exactly what it installs. @@ -231,4 +216,4 @@ def _plan_from_depots( if isinstance(depot_language, str) and depot_language != language: continue required.add(depot_id) - return DepotPlan(required=required, shared=shared) + return DepotPlan(required=required, shared=shared, shared_owner=shared_owner) diff --git a/_installer_import/steam_installer/appinfo_cache.py b/_installer_import/steam_installer/appinfo_cache.py new file mode 100644 index 0000000..ac5eec6 --- /dev/null +++ b/_installer_import/steam_installer/appinfo_cache.py @@ -0,0 +1,115 @@ +"""Read app info from the Steam client's own cache, `appcache/appinfo.vdf`. + +This replaced `steamcmd +app_info_print`. On 2026-11-22 all 11 compared +fields for app 1201270 (name, installdir, build id, change number, depot +gids, ...) matched steamcmd's answer exactly. The cache is only as fresh as +Steam's last run, which is why callers check its gids against what +depotdownloader actually fetched. + +Layout (magic 0x07564429): u32 magic, u32 universe, i64 offset of a string +table; then per app: u32 appid, u32 size, `size` bytes made of a 60-byte +header and binary key-values whose keys are u32 indexes into the string +table. Appid 0 ends the list. The table is a u32 count followed by +NUL-terminated strings. + +Every scalar comes back as a string, the way `app_info_print` printed it: +the rest of the tool compares `== "1"` and `!= str(appid)`, and an int there +would silently change which depots count. +""" + +from __future__ import annotations + +import struct +from pathlib import Path + +from steam_installer.library import find_steam_root + +MAGIC = 0x07564429 +_HEADER = 60 +FIX = "start Steam once, let it sync, rerun" + +# Fixed-width value types: KV type byte -> struct format. +_SCALARS = {2: " Path: + return find_steam_root() / "appcache" / "appinfo.vdf" + + +def load(path: Path | None = None) -> dict[int, dict]: + """Every app in the cache, appid -> its `appinfo` block. + + Parsed once per file version: four lookups in one run would otherwise + parse the ~100 MB file four times. + """ + path = path or default_path() + try: + stat = path.stat() + except OSError as exc: + raise CacheError(f"cannot read Steam's app cache {path}: {FIX}") from exc + key = (stat.st_mtime_ns, stat.st_size) + cached = _memo.get(path) + if cached is not None and cached[0] == key: + return cached[1] + apps = _parse(path.read_bytes()) + _memo[path] = (key, apps) + return apps + + +def _parse(buf: bytes) -> dict[int, dict]: + magic, _universe, table_offset = struct.unpack_from(" list[str]: + (count,) = struct.unpack_from(" tuple[dict, int]: + out: dict = {} + while True: + kind = buf[pos] + pos += 1 + if kind == 8: + return out, pos + (index,) = struct.unpack_from(" argparse.ArgumentParser: "steamcmd leaves behind, and clear them away." ), ) - parser.add_argument( - "--skip-steamcmd", - action="store_true", - help=( - "Finish the manifest directly from depotdownloader's own cached data " - "instead of running steamcmd's app_update. Steamcmd's install-script " - "step has shown two separate failure modes (a confirmed segfault, " - "and a silent no-op run); this is the same direct-write approach " - "already confirmed to work." - ), - ) parser.add_argument( "--verbose", action="store_true", @@ -176,7 +165,7 @@ def _run(args: argparse.Namespace, log: runlog.RunLog) -> int: return _repair_ghosts(log) try: - deps.require(skip_steamcmd=args.skip_steamcmd) + deps.require() resolved = target.resolve_target( args.target, args.appid, interactive=interactive ) diff --git a/_installer_import/steam_installer/deps.py b/_installer_import/steam_installer/deps.py index ee95e39..f9cb3f0 100644 --- a/_installer_import/steam_installer/deps.py +++ b/_installer_import/steam_installer/deps.py @@ -8,7 +8,6 @@ from pathlib import Path TOOL_HINTS = { "depotdownloader": "Run ./install.sh, which fetches it from the SteamRE release.", - "steamcmd": "Run ./install.sh, or: yay -S steamcmd", } @@ -32,17 +31,19 @@ def depotdownloader_bin() -> str: return "depotdownloader" -def missing_tools(*, skip_steamcmd: bool) -> list[str]: - """Which required binaries are not on PATH, in the order they'd be used.""" +def missing_tools() -> list[str]: + """Which required binaries are not on PATH, in the order they'd be used. + + steamcmd is not one of them: app info comes from Steam's own cache and + the manifest is written by this tool. + """ needed = ["depotdownloader"] - if not skip_steamcmd: - needed.append("steamcmd") resolved = {"depotdownloader": depotdownloader_bin()} return [tool for tool in needed if shutil.which(resolved.get(tool, tool)) is None] -def require(*, skip_steamcmd: bool) -> None: - missing = missing_tools(skip_steamcmd=skip_steamcmd) +def require() -> None: + missing = missing_tools() if missing: details = "; ".join( f"`{tool}` not found -- {TOOL_HINTS[tool]}" for tool in missing diff --git a/_installer_import/steam_installer/finalize.py b/_installer_import/steam_installer/finalize.py index 663ad7a..d957f83 100644 --- a/_installer_import/steam_installer/finalize.py +++ b/_installer_import/steam_installer/finalize.py @@ -1,8 +1,9 @@ """Hand-finish the appmanifest, but only when every precondition really holds. Split out of cli.py so the entry point stays under the repo's 250-line cap. -The logic is unchanged: it patches a manifest steamcmd or Steam already -wrote, and refuses rather than fabricating one. +It patches the manifest Steam (or the backlog enforcer) already wrote, and +writes one from scratch when none exists -- steamcmd, which used to create +it, is no longer run. Either way every precondition is checked first. """ from __future__ import annotations @@ -21,32 +22,22 @@ def finish( target: Path, depot_plan: appinfo.DepotPlan, expected_build_id: str | None, - skipped_steamcmd: bool, ) -> int: logger = log.logger manifest_path = library.steamapps / f"appmanifest_{metadata.appid}.acf" - if not manifest_path.is_file(): - # Reachable two ways, so don't blame steamcmd unconditionally: under - # --skip-steamcmd it never runs at all, and nothing else in this tool - # creates a manifest from scratch. - logger.error( - "No %s exists, so there is nothing to finish. This tool only ever " - "patches a manifest Steam or steamcmd already wrote; it never " - "fabricates one. %s", - manifest_path, - ( - "Re-run without --skip-steamcmd so steamcmd creates it (it writes " - "one even when it crashes)." - if skipped_steamcmd - else "steamcmd writes one even when it crashes, so its absence " - "means steamcmd never ran far enough." - ), + seed = None + if manifest_path.is_file(): + app_state = recognize.read_app_state(manifest_path) + else: + # Nothing on disk to patch. Build the manifest in memory and write it + # only once every check below has passed: a StateFlags=4 file left + # behind by a refused run would make Steam fetch the whole game. + logger.info("No %s yet -- writing one from scratch.", manifest_path.name) + seed = manifest_write.seed_app_state( + metadata.appid, metadata.name, target.name, depot_plan.shared_owner ) - logger.info("Run log (send this): %s", log.path) - return 1 - - app_state = recognize.read_app_state(manifest_path) + app_state = seed problems = finish_check.check_ready_to_finish( install_dir=target, app_state=app_state, @@ -77,6 +68,7 @@ def finish( install_dir=target, now_unix=int(time.time()), dlc_depots=depot_plan.dlc, + seed=seed, # Proven depot by depot, so it outranks whatever the manifest says. build_id=( expected_build_id diff --git a/_installer_import/steam_installer/finish_check.py b/_installer_import/steam_installer/finish_check.py index 18e372e..a455c02 100644 --- a/_installer_import/steam_installer/finish_check.py +++ b/_installer_import/steam_installer/finish_check.py @@ -66,8 +66,10 @@ def check_ready_to_finish( ) if stale: problems.append( - f"depot(s) {', '.join(stale)} hold a manifest the public branch no " - "longer ships -- stale content" + f"depot(s) {', '.join(stale)} hold a manifest other than the public " + "gid in Steam's local app cache -- stale content. depotdownloader " + "always fetches the current build, so the cache is the stale side: " + "start Steam once, let it sync, rerun" ) if build_proven(cached, required_depots, expected_manifests): # The manifest's buildid only ever stood in for this proof. A stub diff --git a/_installer_import/steam_installer/flow.py b/_installer_import/steam_installer/flow.py index 3b6fcef..fb27e8d 100644 --- a/_installer_import/steam_installer/flow.py +++ b/_installer_import/steam_installer/flow.py @@ -15,7 +15,6 @@ from steam_installer import ( manifest_write, recognize, runlog, - steamcmd_validate, steamctl, takeover, ) @@ -149,50 +148,6 @@ def _download_and_recognize( ): logger.debug("depot %s -> manifest %s", depot_id, mid) - if not args.skip_steamcmd: - logger.info("Reconciling with steamcmd (validate)...") - log.section("steamcmd app_update validate") - result = steamcmd_validate.run_steamcmd_validate( - metadata.appid, username, logger - ) - log.file_contents("Manifest after steamcmd", manifest_path) - - # Never trust steamcmd's exit code alone -- verify the manifest actually - # reflects a real install before believing it. A "clean" exit has - # produced a manifest steamcmd itself hadn't finished updating before. - if ( - not result.crashed_on_install_script - and recognize.manifest_reflects_real_install( - library, metadata.appid, install_target - ) - ): - # The manifest being valid does not mean steamcmd is why. It can - # already have been correct on arrival while steamcmd failed to even - # log in -- reporting that as "steamcmd finished cleanly" credits a - # step that did nothing and hides a real authentication failure. - if result.returncode == 0: - logger.info("steamcmd finished cleanly -- Steam should show Play.") - else: - logger.warning( - "steamcmd exited %d, but %s is already a complete install " - "-- leaving it as is. Steam should show Play.", - result.returncode, - manifest_path.name, - ) - logger.info("Run log: %s", log.path) - return 0 - - if result.crashed_on_install_script: - logger.info( - "steamcmd hit the known install-script crash; finishing the manifest manually..." - ) - else: - logger.info( - "steamcmd exited %d but the manifest doesn't reflect a real install; " - "checking whether it is safe to finish it manually...", - result.returncode, - ) - return finish( log, library, @@ -200,5 +155,4 @@ def _download_and_recognize( install_target, depot_plan, expected_build_id, - skipped_steamcmd=args.skip_steamcmd, ) diff --git a/_installer_import/steam_installer/manifest_write.py b/_installer_import/steam_installer/manifest_write.py index 307aa15..bb00d3f 100644 --- a/_installer_import/steam_installer/manifest_write.py +++ b/_installer_import/steam_installer/manifest_write.py @@ -1,10 +1,10 @@ """Every write this tool makes to an appmanifest, and only from verified values. -Split out of recognize.py (250-line cap). When steamcmd's validate cannot -finish the job (see :mod:`steamcmd_validate`), the manifest is finished by hand +Split out of recognize.py (250-line cap). The manifest is finished by hand using ONLY values already verified: the exact manifest IDs from -depotdownloader's own local cache (never guessed), and the real on-disk size -measured directly. +depotdownloader's own local cache (never guessed), the build those prove, and +the real on-disk size measured directly. When no manifest exists it is +written from scratch with the 15 keys Steam accepted on 2026-11-22. Do NOT leave the download bookkeeping for Steam to tidy up on its next launch. An earlier version did, on the belief that Steam self-corrects; it does not. A @@ -26,16 +26,15 @@ from steam_installer.library import SteamLibrary def ensure_installdir_correct( library: SteamLibrary, appid: int, install_dir: Path ) -> None: - """Patch a stale `installdir` field BEFORE running steamcmd, not just after. + """Patch a stale `installdir` field before anything else, not just after. Confirmed necessary: a prior, unrelated Steam GUI glitch left this app's - manifest pointing at a different folder name. steamcmd's own validate run - didn't correct it despite a clean exit, and relaunching the Steam client - read that same stale field and wiped the correctly-downloaded folder as + manifest pointing at a different folder name, and relaunching the Steam + client read that stale field and wiped the correctly-downloaded folder as "orphaned" while starting a fresh download into the wrong one. Fixing it - proactively, before steamcmd ever runs, closes that window entirely. - Does nothing if no manifest exists yet -- steamcmd's own app_update writes - one in that case, and it derives installdir from the app's metadata. + first, as soon as Steam is closed, closes that window entirely. Does + nothing if no manifest exists yet: the one written from scratch takes + installdir from the real install folder. """ manifest_path = library.steamapps / f"appmanifest_{appid}.acf" if not manifest_path.is_file(): @@ -66,6 +65,39 @@ def measure_content_size(install_dir: Path) -> int: return total +def seed_app_state( + appid: int, name: str, installdir: str, shared_owner: dict[str, str] +) -> dict: + """The skeleton of a from-scratch manifest, in the key order Steam writes. + + These 15 keys are the proven minimum: Steam logged "Fully Installed,App + Running", queued no download and added LastOwner, UserConfig and the rest + itself. Placeholders here are overwritten by `finish_manifest_manually`. + SharedDepots is only added for a game that has some (fixture-tested only). + """ + state: dict = { + "appid": str(appid), + "universe": "1", + "name": name, + "StateFlags": "4", + "installdir": installdir, + "LastUpdated": "0", + "SizeOnDisk": "0", + "StagingSize": "0", + "buildid": "0", + "UpdateResult": "0", + # All four at 0: counters that disagree made Steam download a game again. + "BytesToDownload": "0", + "BytesDownloaded": "0", + "BytesToStage": "0", + "BytesStaged": "0", + "InstalledDepots": {}, + } + if shared_owner: + state["SharedDepots"] = dict(shared_owner) + return state + + def finish_manifest_manually( library: SteamLibrary, appid: int, @@ -73,26 +105,27 @@ def finish_manifest_manually( now_unix: int, dlc_depots: dict[str, str] | None = None, build_id: str | None = None, + seed: dict | None = None, ) -> Path: """Patch appmanifest_.acf to StateFlags=4 using only verified-known values. - Requires steamcmd to have already written a partial manifest (even a crashed - run does) -- that's where buildid and SharedDepots come from. This never - fabricates a manifest from nothing. + Patches the manifest on disk, or -- when there is none -- writes `seed` + (from :func:`seed_app_state`) instead. With neither it refuses. `dlc_depots` maps a DLC depot to its DLC app id. Steam's own manifests record each DLC depot with that `dlcappid`, so it is written the same way. `build_id`, when given, is a build the caller proved from the cached depot - manifests; it replaces the one Steam or steamcmd recorded. + manifests; it replaces the one Steam recorded. """ manifest_path = library.steamapps / f"appmanifest_{appid}.acf" - if not manifest_path.is_file(): + existed = manifest_path.is_file() + if not existed and seed is None: raise recognize.RecognizeError( - f"No existing {manifest_path.name} to finish -- run steamcmd's " - "app_update at least once first (even a crashed run writes one)." + f"No existing {manifest_path.name} to finish and no seed to write" ) - data = vdf.parse(manifest_path.read_text()) - app_state = data.get("AppState", {}) + app_state = ( + vdf.parse(manifest_path.read_text()).get("AppState", {}) if existed else seed + ) if not isinstance(app_state, dict): raise recognize.RecognizeError( f"{manifest_path} is not a valid AppState manifest" @@ -105,7 +138,7 @@ def finish_manifest_manually( ) size_on_disk = measure_content_size(install_dir) - # "0" means "no real build installed yet" (steamcmd/Steam use it as a + # "0" means "no real build installed yet" (Steam uses it as a # sentinel) -- a bare `or` treats that non-empty string as truthy and # never falls through to TargetBuildID, leaving a stale/wrong buildid. existing_buildid = app_state.get("buildid") @@ -145,7 +178,8 @@ def finish_manifest_manually( app_state.pop("DlcDownloads", None) settle_download_counters(app_state) - backup_once(manifest_path) + if existed: + backup_once(manifest_path) write_atomic(manifest_path, vdf.dumps("AppState", app_state)) return manifest_path diff --git a/_installer_import/steam_installer/steamcmd_validate.py b/_installer_import/steam_installer/steamcmd_validate.py deleted file mode 100644 index b62f70d..0000000 --- a/_installer_import/steam_installer/steamcmd_validate.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Reconcile an install with Steam's own bookkeeping via `steamcmd ... validate`. - -Split out of recognize.py (250-line cap). `app_update validate` hash-checks -existing files against the depot manifest and only fetches real deltas -- -confirmed live against a real 90+ GB install, going from ~24% to ~41% credited -in one pass without redownloading from scratch. - -Some titles ship a post-install script (installscript.vdf) that steamcmd runs -through a private, headless Proton setup. On at least one real title this hit a -confirmed steamcmd bug -- a stack-overflow segfault in the closed-source -steamclient.so error-handling path after the script evaluator reports failure -(the same stack frame repeating dozens of times). There's no known fix; the -caller detects that crash and finishes the manifest by hand instead -(:mod:`manifest_write`). -""" - -from __future__ import annotations - -import logging -import signal -from dataclasses import dataclass - -from steam_installer import runlog - - -@dataclass -class ValidateResult: - completed: bool - crashed_on_install_script: bool - returncode: int - output: str = "" - - -def run_steamcmd_validate( - appid: int, - username: str, - logger: logging.Logger, -) -> ValidateResult: - """Run steamcmd's app_update ... validate. - - Takes no install dir on purpose. `force_install_dir` sets a Steam *library - root*, not an install directory: pointing it at the game folder made - steamcmd look for manifests in `/steamapps/`, find none ("Loaded 0 - apps from install folder"), create that stray directory, and then fail the - app with "Invalid platform" because it had no depot context for it. The app - already lives in a registered library, so letting steamcmd use the real one - is both correct and sufficient. - - Output is teed through a pty (see runlog) rather than captured: steamcmd - prompts for a password and possibly a Steam Guard code on stdin, and prompt - text written to a captured/buffered pipe never reaches the terminal -- the - process looks hung with no visible request for input. - - The confirmed steamcmd install-script bug (see module docstring) kills - the process with SIGSEGV -- Python reports that as returncode -11 on - POSIX. Checking the signal directly is more reliable than grepping - captured text for a message string. - """ - cmd = [ - "steamcmd", - # A Linux steamcmd refuses a Windows-only app with "Invalid platform" - # unless told which platform to resolve depots for. Must precede +login. - "+@sSteamCmdForcePlatformType", - "windows", - "+login", - username, - "+app_update", - str(appid), - "validate", - "+quit", - ] - returncode, output = runlog.run_logged(cmd, logger, timeout=3600) - crashed = returncode == -signal.SIGSEGV - return ValidateResult( - completed=returncode == 0, - crashed_on_install_script=crashed, - returncode=returncode, - output=output, - ) diff --git a/_installer_import/tests/_appinfo_blob.py b/_installer_import/tests/_appinfo_blob.py new file mode 100644 index 0000000..118926f --- /dev/null +++ b/_installer_import/tests/_appinfo_blob.py @@ -0,0 +1,49 @@ +"""Build a binary appinfo.vdf (magic 0x07564429) for tests. + +The writer mirrors what Steam stores: a header, one entry per app (appid, +size, 60 header bytes, binary key-values whose keys index a string table), +a zero appid terminator, then the string table itself. +""" + +from __future__ import annotations + +import struct + +MAGIC = 0x07564429 + + +def _kv(data: dict, strings: dict[str, int]) -> bytes: + out = bytearray() + for key, value in data.items(): + index = strings.setdefault(key, len(strings)) + if isinstance(value, dict): + out += b"\x00" + struct.pack(" bytes: + """Serialize `{appid: appinfo-dict}`; each dict is wrapped in "appinfo".""" + strings: dict[str, int] = {} + body = bytearray() + for appid, data in apps.items(): + kv = _kv({"appinfo": data}, strings) + header = struct.pack(""` header there is no app info to parse.""" - monkeypatch.setattr( - appinfo.subprocess, - "run", - lambda cmd, **kw: _FakeCompleted("Logging in user 'anonymous'...OK\n"), - ) - with pytest.raises(appinfo.AppInfoError, match="did not return app info"): +def test_app_missing_from_the_cache_fails_with_the_fix(cache): + cache({1771300: KCD2}) + with pytest.raises(appinfo.AppInfoError, match="start Steam once, let it sync, rerun"): appinfo.get_metadata(999999999) -def test_fetch_app_block_raises_when_block_is_not_a_mapping(monkeypatch): - """Defensive guard against a shape vdf.parse cannot currently produce (it - always yields a dict for a `{ ... }` block). Exercised by injecting a parser - that returns a scalar, so the guard stays correct if vdf ever changes.""" +def test_unreadable_cache_fails_with_the_fix(tmp_path, monkeypatch): monkeypatch.setattr( - appinfo.subprocess, - "run", - lambda cmd, **kw: _FakeCompleted('"1771300"\n{\n}\n'), + appinfo.appinfo_cache, "default_path", lambda: tmp_path / "missing.vdf" ) - monkeypatch.setattr(appinfo.vdf, "parse", lambda text: {"1771300": "scalar"}) + with pytest.raises(appinfo.AppInfoError, match="start Steam once"): + appinfo.get_metadata(1) - with pytest.raises(appinfo.AppInfoError, match="unexpected app_info_print"): - appinfo.get_metadata(1771300) + +def test_dlc_missing_from_the_cache_counts_as_not_owned(cache): + """Steam caches app info for what the account owns; on 2026-11-22 only 1828 + of 7102 listed DLC ids were in the cache. Refusing on a missing one would + block installs that worked with steamcmd.""" + game = {**KCD2, "extended": {"listofdlc": "500,600"}} + dlc = {"depots": {"501": {"manifests": {"public": {"gid": "9"}}}}} + cache({1771300: game, 500: dlc}) + assert appinfo.get_dlc_plans(1771300) == {500: {"501": "9"}} def test_get_build_id_handles_non_dict_sections(monkeypatch): - for text in ( - '"1"\n{\n\t"depots"\t\t"scalar"\n}\n', - '"1"\n{\n\t"depots"\n\t{\n\t\t"branches"\t\t"scalar"\n\t}\n}\n', - '"1"\n{\n\t"depots"\n\t{\n\t\t"branches"\n\t\t{\n\t\t\t"public"\t\t"x"\n\t\t}\n\t}\n}\n', + for block in ( + {"depots": "scalar"}, + {"depots": {"branches": "scalar"}}, + {"depots": {"branches": {"public": "x"}}}, ): - monkeypatch.setattr( - appinfo.subprocess, "run", lambda cmd, _t=text, **kw: _FakeCompleted(_t) - ) + _app_block(monkeypatch, block) assert appinfo.get_build_id(1) is None def test_get_build_id_returns_none_when_buildid_not_a_string(monkeypatch): - text = '"1"\n{\n\t"depots"\n\t{\n\t\t"branches"\n\t\t{\n\t\t\t"public"\n\t\t\t{\n\t\t\t\t"buildid"\n\t\t\t\t{\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n' - monkeypatch.setattr( - appinfo.subprocess, "run", lambda cmd, **kw: _FakeCompleted(text) - ) + _app_block(monkeypatch, {"depots": {"branches": {"public": {"buildid": {}}}}}) assert appinfo.get_build_id(1) is None - diff --git a/_installer_import/tests/test_appinfo_cache.py b/_installer_import/tests/test_appinfo_cache.py new file mode 100644 index 0000000..18d6dc5 --- /dev/null +++ b/_installer_import/tests/test_appinfo_cache.py @@ -0,0 +1,81 @@ +"""Steam's local binary app-info cache, read without steamcmd.""" + +import pytest +from steam_installer import appinfo_cache + +from _appinfo_blob import encode + + +def _write(tmp_path, apps): + path = tmp_path / "appinfo.vdf" + path.write_bytes(encode(apps)) + return path + + +def test_reads_every_app_and_unwraps_appinfo(tmp_path): + path = _write(tmp_path, {10: {"common": {"name": "A"}}, 20: {"common": {}}}) + apps = appinfo_cache.load(path) + assert set(apps) == {10, 20} + assert apps[10] == {"common": {"name": "A"}} + + +def test_scalars_come_back_as_strings_like_app_info_print(tmp_path): + """The rest of the tool compares `== "1"` and `!= str(appid)`; an int here + would silently drop a DLC app's own depots.""" + data = {"depots": {"5": {"dlcappid": 7, "optional": 1, "f": 1.5, + "gid": (7, 2**63 + 5), "neg": (10, -3)}}} + apps = appinfo_cache.load(_write(tmp_path, {7: data})) + assert apps[7]["depots"]["5"] == { + "dlcappid": "7", "optional": "1", "f": "1.5", + "gid": str(2**63 + 5), "neg": "-3", + } + + +def test_parse_is_cached_until_the_file_changes(tmp_path, monkeypatch): + path = _write(tmp_path, {1: {"common": {"name": "old"}}}) + assert appinfo_cache.load(path)[1]["common"]["name"] == "old" + calls = [] + real = appinfo_cache._parse + monkeypatch.setattr(appinfo_cache, "_parse", lambda b: calls.append(1) or real(b)) + appinfo_cache.load(path) + assert calls == [] + path.write_bytes(encode({1: {"common": {"name": "newer"}}})) + assert appinfo_cache.load(path)[1]["common"]["name"] == "newer" + assert calls == [1] + + +def test_wrong_magic_is_refused(tmp_path): + path = tmp_path / "appinfo.vdf" + path.write_bytes(b"\x28\x44\x56\x07" + b"\0" * 20) + with pytest.raises(appinfo_cache.CacheError, match="unsupported"): + appinfo_cache.load(path) + + +def test_unknown_value_type_is_refused(tmp_path): + blob = bytearray(encode({1: {"common": {"name": "A"}}})) + blob[16 + 8 + 60] = 9 # first KV type byte + path = tmp_path / "appinfo.vdf" + path.write_bytes(bytes(blob)) + with pytest.raises(appinfo_cache.CacheError, match="value type 9"): + appinfo_cache.load(path) + + +def test_missing_file_names_the_fix(tmp_path): + with pytest.raises(appinfo_cache.CacheError, match="start Steam once"): + appinfo_cache.load(tmp_path / "nope.vdf") + + +def test_entry_without_appinfo_wrapper_is_taken_as_is(tmp_path): + import _appinfo_blob + + blob = _appinfo_blob.encode({3: {"x": "y"}}) + # Rename the "appinfo" wrapper key in the string table. + blob = blob.replace(b"appinfo\x00", b"appinfX\x00") + path = tmp_path / "appinfo.vdf" + path.write_bytes(blob) + assert appinfo_cache.load(path)[3] == {"appinfX": {"x": "y"}} + + +def test_default_path_is_under_the_steam_root(monkeypatch, tmp_path): + monkeypatch.setattr(appinfo_cache, "find_steam_root", lambda: tmp_path) + assert appinfo_cache.default_path() == tmp_path / "appcache" / "appinfo.vdf" diff --git a/_installer_import/tests/test_cli.py b/_installer_import/tests/test_cli.py index 881c92b..afd6e97 100644 --- a/_installer_import/tests/test_cli.py +++ b/_installer_import/tests/test_cli.py @@ -1,7 +1,7 @@ """End-to-end-ish tests for the CLI's decision logic. The bug that motivated all of this lived here, not in the manifest writer: -steamcmd died instantly having done nothing, and main() still wrote +a step died instantly having done nothing, and main() still wrote StateFlags=4, printed "launch Steam and confirm Play works" and returned 0. These tests pin the fail-closed behaviour. """ @@ -13,21 +13,19 @@ from steam_installer import ( depotdl, manifest_write, recognize, - steamcmd_validate, vdf, ) -from _cli_helpers import ARGV, _stub_download, _stub_steamcmd +from _cli_helpers import ARGV, _stub_download pytestmark = pytest.mark.usefixtures("cli_logs") -def test_dead_steamcmd_run_still_finishes_when_content_is_present(steam, monkeypatch): - """steamcmd failing is survivable -- the content is what matters -- but the - manifest must come out consistent, not merely flagged installed.""" +def test_downloaded_content_finishes_the_manifest(steam, monkeypatch): + """The content is what matters, and the manifest must come out consistent, + not merely flagged installed.""" library, install_dir = steam _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) assert cli.main(ARGV) == 0 @@ -42,7 +40,6 @@ def test_refuses_when_a_required_depot_never_downloaded(steam, monkeypatch): app installed would be a fabricated success.""" library, install_dir = steam _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) monkeypatch.setattr( appinfo, "get_depot_plan", @@ -58,7 +55,6 @@ def test_refuses_when_a_required_depot_never_downloaded(steam, monkeypatch): def test_refuses_when_depotdownloader_fails(steam, monkeypatch): library, _ = steam _stub_download(monkeypatch, returncode=1) - _stub_steamcmd(monkeypatch, returncode=0) assert cli.main(ARGV) == 1 app_state = recognize.read_app_state(library.steamapps / "appmanifest_1771300.acf") @@ -79,79 +75,6 @@ def test_declining_to_close_steam_aborts_before_downloading(steam, monkeypatch): assert called == [] -def test_clean_steamcmd_run_short_circuits(steam, monkeypatch): - """If steamcmd genuinely finished the job, don't hand-patch anything.""" - library, install_dir = steam - _stub_download(monkeypatch) - - def fake_validate(appid, username, logger): - (library.steamapps / "appmanifest_1771300.acf").write_text( - vdf.dumps( - "AppState", - { - "StateFlags": "4", - "installdir": "KingdomComeDeliverance2", - "InstalledDepots": {"1771303": {"manifest": "1"}}, - "BytesToDownload": "1", - "BytesDownloaded": "1", - }, - ) - ) - return steamcmd_validate.ValidateResult(True, False, 0) - - monkeypatch.setattr(steamcmd_validate, "run_steamcmd_validate", fake_validate) - called = [] - monkeypatch.setattr( - manifest_write, - "finish_manifest_manually", - lambda **kw: called.append(1), - ) - - assert cli.main(ARGV) == 0 - assert called == [] - - -def test_a_complete_install_after_a_failed_steamcmd_is_not_credited_to_it( - steam, monkeypatch, capsys -): - """A valid manifest does not mean steamcmd is why it is valid. - - steamcmd can fail to even log in while the install was already complete on - arrival. Reporting that as "finished cleanly" credits a step that did - nothing and hides a real authentication failure, so the non-zero exit has - to say so out loud. - """ - library, _ = steam - _stub_download(monkeypatch) - - def fake_validate(appid, username, logger): - (library.steamapps / "appmanifest_1771300.acf").write_text( - vdf.dumps( - "AppState", - { - "StateFlags": "4", - "installdir": "KingdomComeDeliverance2", - "InstalledDepots": {"1771303": {"manifest": "1"}}, - "BytesToDownload": "1", - "BytesDownloaded": "1", - }, - ) - ) - return steamcmd_validate.ValidateResult(True, False, 5) - - monkeypatch.setattr(steamcmd_validate, "run_steamcmd_validate", fake_validate) - called = [] - monkeypatch.setattr( - manifest_write, "finish_manifest_manually", lambda **kw: called.append(1) - ) - - assert cli.main(ARGV) == 0 - out = capsys.readouterr().out - assert "steamcmd exited 5" in out - assert "finished cleanly" not in out - assert called == [] - - def test_user_declining_confirmation_aborts(steam, monkeypatch): monkeypatch.setattr("builtins.input", lambda *a: "n") called = [] diff --git a/_installer_import/tests/test_cli_finish.py b/_installer_import/tests/test_cli_finish.py index 5cf6297..c320d43 100644 --- a/_installer_import/tests/test_cli_finish.py +++ b/_installer_import/tests/test_cli_finish.py @@ -11,36 +11,87 @@ from steam_installer import ( vdf, ) -from _cli_helpers import ARGV, _stub_download, _stub_steamcmd +from _cli_helpers import ARGV, _stub_download pytestmark = pytest.mark.usefixtures("cli_logs") -def test_missing_manifest_under_skip_steamcmd_does_not_blame_steamcmd( - steam, monkeypatch, capsys -): - """--skip-steamcmd never runs steamcmd, so "steamcmd never ran far enough" - would send the user chasing the wrong thing.""" - library, _ = steam - (library.steamapps / "appmanifest_1771300.acf").unlink() +# The 15 keys Steam accepted on 2026-11-22 (A Space for the Unbound): it +# logged "Fully Installed,App Running", queued no download, and added its own +# LastOwner, UserConfig, MountedConfig, ... on exit. +FROM_SCRATCH_KEYS = [ + "appid", "universe", "name", "StateFlags", "installdir", "LastUpdated", + "SizeOnDisk", "StagingSize", "buildid", "UpdateResult", "BytesToDownload", + "BytesDownloaded", "BytesToStage", "BytesStaged", "InstalledDepots", +] + + +def _proven_plan(monkeypatch, install_dir, shared_owner=None): + (gid,) = recognize.discover_cached_depot_manifests(install_dir).values() + monkeypatch.setattr( + appinfo, + "get_depot_plan", + lambda appid, language: appinfo.DepotPlan( + {"1771303"}, set(shared_owner or ()), manifests={"1771303": gid}, + shared_owner=dict(shared_owner or {}), + ), + ) + return gid + + +def test_missing_manifest_is_written_from_scratch(steam, monkeypatch): + """No steamcmd any more, so nothing else creates the manifest.""" + library, install_dir = steam + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.unlink() + gid = _proven_plan(monkeypatch, install_dir) _stub_download(monkeypatch) - assert cli.main([*ARGV, "--skip-steamcmd"]) == 1 + assert cli.main(ARGV) == 0 - out = capsys.readouterr().out - assert "never fabricates one" in out - assert "Re-run without --skip-steamcmd" in out - assert "steamcmd never ran far enough" not in out + app_state = vdf.parse(manifest.read_text())["AppState"] + assert list(app_state) == FROM_SCRATCH_KEYS + assert app_state["appid"] == "1771300" + assert app_state["universe"] == "1" + assert app_state["name"] == "KCD2" + assert app_state["StateFlags"] == "4" + assert app_state["installdir"] == "KingdomComeDeliverance2" + assert app_state["SizeOnDisk"] == "1000" + assert app_state["buildid"] == "23914554" + for key in ("StagingSize", "UpdateResult", "BytesToDownload", + "BytesDownloaded", "BytesToStage", "BytesStaged"): + assert app_state[key] == "0", key + assert app_state["InstalledDepots"] == {"1771303": {"manifest": gid, "size": "0"}} + # Nothing to back up: there was no manifest before. + assert not manifest.with_suffix(".acf.bak").exists() -def test_missing_manifest_with_steamcmd_blames_steamcmd(steam, monkeypatch, capsys): - library, _ = steam - (library.steamapps / "appmanifest_1771300.acf").unlink() +def test_a_shared_depot_is_recorded_under_its_owner_from_scratch(steam, monkeypatch): + """Fixture-only: no shared-depot game was tested live. Steam records a + depot installed under another app as SharedDepots {depot: owner}.""" + library, install_dir = steam + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.unlink() + _proven_plan(monkeypatch, install_dir, shared_owner={"228989": "228980"}) + _stub_download(monkeypatch) + + assert cli.main(ARGV) == 0 + app_state = vdf.parse(manifest.read_text())["AppState"] + assert app_state["SharedDepots"] == {"228989": "228980"} + assert "228989" not in app_state["InstalledDepots"] + + +def test_unproven_content_writes_no_manifest_from_scratch(steam, monkeypatch, capsys): + """Fail closed: without a proven build there is no buildid to write, and a + half-true StateFlags=4 manifest is what made Steam redownload games.""" + library, _ = steam + manifest = library.steamapps / "appmanifest_1771300.acf" + manifest.unlink() _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) assert cli.main(ARGV) == 1 - assert "steamcmd never ran far enough" in capsys.readouterr().out + assert not manifest.exists() + assert "no usable buildid" in capsys.readouterr().out def test_language_defaults_to_english_when_prompt_is_empty(steam, monkeypatch): @@ -56,21 +107,12 @@ def test_language_defaults_to_english_when_prompt_is_empty(steam, monkeypatch): ), ) _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) cli.main(["--appid", "1771300", "--username", "u"]) assert seen["language"] == "english" -def test_reports_the_known_install_script_crash(steam, monkeypatch, capsys): - _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=-11, crashed=True) - - assert cli.main(ARGV) == 0 - assert "known install-script crash" in capsys.readouterr().out - - def test_adopts_staged_content_then_clears_it(steam, monkeypatch, capsys): """Steam's staging tree is adopted into the install dir before downloading, and only cleared once the install is finished -- never discarded up front.""" @@ -79,7 +121,6 @@ def test_adopts_staged_content_then_clears_it(steam, monkeypatch, capsys): staged.mkdir(parents=True) (staged / "big.pak").write_bytes(b"x" * 2_000_000) _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) assert cli.main(ARGV) == 0 @@ -96,7 +137,6 @@ def test_fails_when_written_manifest_does_not_read_back_as_installed( """Claiming success without re-reading the write is the same fail-open mistake in a different place.""" _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) monkeypatch.setattr( recognize, "manifest_reflects_real_install", lambda *a, **kw: False ) @@ -116,7 +156,7 @@ def test_module_is_runnable_as_a_script(coverage_subprocess_env): env=coverage_subprocess_env, ) assert result.returncode == 0 - assert "--skip-steamcmd" in result.stdout + assert "--skip-steamcmd" not in result.stdout def test_a_stub_manifest_is_finished_with_the_proven_build(steam, monkeypatch): @@ -138,7 +178,7 @@ def test_a_stub_manifest_is_finished_with_the_proven_build(steam, monkeypatch): ) _stub_download(monkeypatch) - assert cli.main([*ARGV, "--skip-steamcmd"]) == 0 + assert cli.main(ARGV) == 0 app_state = vdf.parse(manifest.read_text())["AppState"] assert (app_state["StateFlags"], app_state["buildid"]) == ("4", "23914554") @@ -156,5 +196,5 @@ def test_the_install_folder_is_corrected_even_when_the_download_fails( manifest.write_text(vdf.dumps("AppState", state)) _stub_download(monkeypatch, returncode=2) - assert cli.main([*ARGV, "--skip-steamcmd"]) == 1 + assert cli.main(ARGV) == 1 assert recognize.read_app_state(manifest)["installdir"] == "KingdomComeDeliverance2" diff --git a/_installer_import/tests/test_deps.py b/_installer_import/tests/test_deps.py index 1041c7b..c048f02 100644 --- a/_installer_import/tests/test_deps.py +++ b/_installer_import/tests/test_deps.py @@ -5,27 +5,26 @@ import pytest from steam_installer import deps -def test_nothing_missing_when_both_tools_are_on_path(monkeypatch): +def test_nothing_missing_when_depotdownloader_is_on_path(monkeypatch): monkeypatch.setattr(deps.shutil, "which", lambda name: f"/usr/bin/{name}") - assert deps.missing_tools(skip_steamcmd=False) == [] - deps.require(skip_steamcmd=False) + assert deps.missing_tools() == [] + deps.require() -def test_steamcmd_is_not_required_when_it_is_skipped(monkeypatch): +def test_steamcmd_is_never_required(monkeypatch): monkeypatch.setattr( deps.shutil, "which", lambda name: None if name == "steamcmd" else "/usr/bin/dd" ) - assert deps.missing_tools(skip_steamcmd=True) == [] - assert deps.missing_tools(skip_steamcmd=False) == ["steamcmd"] + assert deps.missing_tools() == [] def test_missing_tool_raises_with_the_command_that_fixes_it(monkeypatch): monkeypatch.setattr(deps.shutil, "which", lambda name: None) with pytest.raises(deps.DependencyError) as excinfo: - deps.require(skip_steamcmd=False) + deps.require() message = str(excinfo.value) assert "depotdownloader" in message - assert "steamcmd" in message + assert "steamcmd" not in message assert "./install.sh" in message diff --git a/_installer_import/tests/test_dlc.py b/_installer_import/tests/test_dlc.py index 0ef3652..342e440 100644 --- a/_installer_import/tests/test_dlc.py +++ b/_installer_import/tests/test_dlc.py @@ -6,10 +6,9 @@ own app id; 2060310 is listed in the game's own depots; 2441600 has none. """ import logging -import subprocess import pytest -from _cli_helpers import ARGV, _stub_download, _stub_steamcmd +from _cli_helpers import ARGV, _stub_download from steam_installer import appinfo, cli, depotdl, deps, dlc, recognize, runlog, vdf @@ -42,7 +41,7 @@ DLC_BLOCKS = { def _app_info(monkeypatch, game=GAME, dlc_blocks=DLC_BLOCKS): monkeypatch.setattr(appinfo, "_fetch_app_block", lambda appid: game) monkeypatch.setattr( - appinfo, "_fetch_app_blocks", lambda ids: {i: dlc_blocks[i] for i in ids} + appinfo, "_cached_blocks", lambda ids: {i: dlc_blocks[i] for i in ids} ) @@ -54,7 +53,7 @@ def test_only_dlc_with_depots_of_its_own_needs_a_run(monkeypatch): @pytest.mark.parametrize("game", [{}, {"extended": "scalar"}, {"extended": {}}]) def test_a_game_without_dlc_needs_no_lookup(monkeypatch, game): monkeypatch.setattr(appinfo, "_fetch_app_block", lambda appid: game) - monkeypatch.setattr(appinfo, "_fetch_app_blocks", lambda ids: pytest.fail()) + monkeypatch.setattr(appinfo, "_cached_blocks", lambda ids: pytest.fail()) assert appinfo.get_dlc_plans(1) == {} @@ -70,25 +69,6 @@ def test_depot_plan_tags_the_games_own_dlc_depots(monkeypatch): assert plan.dlc == {"2060310": "2060310"} -def test_app_info_for_many_apps_comes_from_one_steamcmd_call(monkeypatch): - output = ( - 'Loading...\n"2060310"\n{\n\t"common"\n\t{\n\t\t"name"\t"A"\n\t}\n}\n' - '"2138330"\n{\n\t"common"\n\t{\n\t\t"name"\t"B"\n\t}\n}\nUnloading\n' - ) - calls = [] - - def fake_run(cmd, **kwargs): - calls.append(cmd) - return subprocess.CompletedProcess(cmd, 0, stdout=output, stderr="") - - monkeypatch.setattr(appinfo.subprocess, "run", fake_run) - blocks = appinfo._fetch_app_blocks([2138330, 2060310]) - assert blocks[2060310]["common"]["name"] == "A" - assert blocks[2138330]["common"]["name"] == "B" - assert len(calls) == 1 - assert calls[0].count("+app_info_print") == 2 - - def _logged(monkeypatch, returncode, output=""): monkeypatch.setattr( depotdl.runlog, "run_logged", lambda cmd, logger: (returncode, output) @@ -163,7 +143,6 @@ def test_downloaded_dlc_is_required_and_credited_to_its_app(steam, monkeypatch): } manifest.write_text(vdf.dumps("AppState", state)) _stub_download(monkeypatch) - _stub_steamcmd(monkeypatch, returncode=8) monkeypatch.setattr(dlc, "download_owned", lambda *a: _dlc_plan("100389")) assert cli.main(ARGV) == 0 @@ -197,7 +176,7 @@ def test_a_stub_with_own_app_dlc_finishes_with_the_proven_build(steam, monkeypat _stub_download(monkeypatch) monkeypatch.setattr(dlc, "download_owned", lambda *a: _dlc_plan("100389")) - assert cli.main([*ARGV, "--skip-steamcmd"]) == 0 + assert cli.main(ARGV) == 0 assert recognize.read_app_state(manifest)["buildid"] == "23914554" diff --git a/_installer_import/tests/test_finish_check.py b/_installer_import/tests/test_finish_check.py index 3fae028..cf36e38 100644 --- a/_installer_import/tests/test_finish_check.py +++ b/_installer_import/tests/test_finish_check.py @@ -193,6 +193,9 @@ def test_a_stale_depot_manifest_is_refused(tmp_path, monkeypatch): expected_manifests={"1086941": "8253106406752983615"}, ) assert any("1086941" in p and "stale" in p for p in problems) + # The cache is the side that lags: depotdownloader always fetches the + # current build, so the fix is to let Steam refresh its cache. + assert any("start Steam once, let it sync, rerun" in p for p in problems) def test_a_proven_build_still_needs_a_buildid_to_record(tmp_path, monkeypatch): diff --git a/_installer_import/tests/test_manifest_finish.py b/_installer_import/tests/test_manifest_finish.py index 75dea03..087e999 100644 --- a/_installer_import/tests/test_manifest_finish.py +++ b/_installer_import/tests/test_manifest_finish.py @@ -15,7 +15,7 @@ def test_finish_manifest_manually_requires_existing_manifest(tmp_path): library = SteamLibrary(tmp_path / "Steam") install_dir = tmp_path / "Steam" / "steamapps" / "common" / "Game" install_dir.mkdir(parents=True) - with pytest.raises(RecognizeError, match="run steamcmd"): + with pytest.raises(RecognizeError, match="no seed to write"): finish_manifest_manually(library, 1771300, install_dir, now_unix=1000) diff --git a/_installer_import/tests/test_steamcmd_validate.py b/_installer_import/tests/test_steamcmd_validate.py deleted file mode 100644 index 3a1c474..0000000 --- a/_installer_import/tests/test_steamcmd_validate.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging -import signal - -from steam_installer.steamcmd_validate import run_steamcmd_validate - - -def test_run_steamcmd_validate_argv_and_result(monkeypatch): - seen = {} - - def fake_run_logged(cmd, logger, timeout=None): - seen["cmd"] = cmd - seen["timeout"] = timeout - return 0, "all good" - - monkeypatch.setattr( - "steam_installer.steamcmd_validate.runlog.run_logged", fake_run_logged - ) - - result = run_steamcmd_validate(1771300, "krzys2000", logging.getLogger("t")) - - cmd = seen["cmd"] - # The platform flag must precede +login, and no force_install_dir may appear: - # it sets a LIBRARY ROOT and pointing it at the game dir caused the original - # "Invalid platform" failure. - assert cmd.index("+@sSteamCmdForcePlatformType") < cmd.index("+login") - assert "force_install_dir" not in " ".join(cmd) - assert cmd[cmd.index("+app_update") + 1 : cmd.index("+app_update") + 3] == [ - "1771300", - "validate", - ] - assert result.completed and result.returncode == 0 - assert result.output == "all good" - assert not result.crashed_on_install_script - - -def test_run_steamcmd_validate_detects_the_install_script_segfault(monkeypatch): - """The known steamcmd bug kills the process with SIGSEGV; checking the - signal beats grepping captured text for a message string.""" - monkeypatch.setattr( - "steam_installer.steamcmd_validate.runlog.run_logged", - lambda cmd, logger, timeout=None: (-signal.SIGSEGV, ""), - ) - - result = run_steamcmd_validate(1, "u", logging.getLogger("t")) - - assert result.crashed_on_install_script - assert not result.completed From c1b6fdd16f07e91313dc3c2ec821965a1aef15c4 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 22 Nov 2026 14:57:48 +0100 Subject: [PATCH 19/24] Map safe games to a FOSS Proton Install the pinned proton-cachyos-native build (signature and sha256 checked, .INSTALL never run), decide per game whether it is safe, write the CompatToolMapping as text, add --map-installed and --valve-proton, and drop steamcmd from install.sh. Co-Authored-By: Claude Opus 5.5 --- _installer_import/.pre-commit-config.yaml | 7 +- _installer_import/README.md | 92 ++++++-- _installer_import/install.sh | 43 +++- _installer_import/pyproject.toml | 4 +- _installer_import/run.sh | 5 +- _installer_import/scripts/deploy.sh | 2 +- .../steam_installer/anticheat.py | 57 +++++ _installer_import/steam_installer/appinfo.py | 1 - _installer_import/steam_installer/cli.py | 15 +- .../steam_installer/compat_apply.py | 74 ++++++ .../steam_installer/compat_decide.py | 108 +++++++++ .../steam_installer/compat_map.py | 208 +++++++++++++++++ _installer_import/steam_installer/config.py | 5 +- .../data/cachyos_signing_key.asc | 53 +++++ .../data/compat_overrides.json | 1 + .../steam_installer/data/proton-native.lock | 7 + .../steam_installer/finish_check.py | 2 +- _installer_import/steam_installer/flow.py | 17 +- .../steam_installer/map_installed.py | 125 +++++++++++ .../steam_installer/proton_native.py | 173 ++++++++++++++ .../steam_installer/proton_pkg.py | 208 +++++++++++++++++ .../steam_installer/recognize.py | 4 +- _installer_import/steam_installer/runlog.py | 12 +- _installer_import/steam_installer/target.py | 4 +- _installer_import/steam_installer/vdf.py | 4 +- _installer_import/tests/_proton_fixture.py | 121 ++++++++++ _installer_import/tests/conftest.py | 31 +++ _installer_import/tests/test_appinfo.py | 7 + _installer_import/tests/test_appinfo_cache.py | 4 +- _installer_import/tests/test_cli_finish.py | 4 +- _installer_import/tests/test_compat_decide.py | 164 ++++++++++++++ _installer_import/tests/test_compat_map.py | 198 ++++++++++++++++ _installer_import/tests/test_install_sh.py | 45 ++++ _installer_import/tests/test_map_installed.py | 197 ++++++++++++++++ _installer_import/tests/test_proton_native.py | 211 ++++++++++++++++++ .../tests/test_proton_native_cli.py | 58 +++++ _installer_import/tests/test_proton_pkg.py | 98 ++++++++ 37 files changed, 2303 insertions(+), 66 deletions(-) create mode 100644 _installer_import/steam_installer/anticheat.py create mode 100644 _installer_import/steam_installer/compat_apply.py create mode 100644 _installer_import/steam_installer/compat_decide.py create mode 100644 _installer_import/steam_installer/compat_map.py create mode 100644 _installer_import/steam_installer/data/cachyos_signing_key.asc create mode 100644 _installer_import/steam_installer/data/compat_overrides.json create mode 100644 _installer_import/steam_installer/data/proton-native.lock create mode 100644 _installer_import/steam_installer/map_installed.py create mode 100644 _installer_import/steam_installer/proton_native.py create mode 100644 _installer_import/steam_installer/proton_pkg.py create mode 100644 _installer_import/tests/_proton_fixture.py create mode 100644 _installer_import/tests/test_compat_decide.py create mode 100644 _installer_import/tests/test_compat_map.py create mode 100644 _installer_import/tests/test_install_sh.py create mode 100644 _installer_import/tests/test_map_installed.py create mode 100644 _installer_import/tests/test_proton_native.py create mode 100644 _installer_import/tests/test_proton_native_cli.py create mode 100644 _installer_import/tests/test_proton_pkg.py diff --git a/_installer_import/.pre-commit-config.yaml b/_installer_import/.pre-commit-config.yaml index ce3b75d..4cf791d 100644 --- a/_installer_import/.pre-commit-config.yaml +++ b/_installer_import/.pre-commit-config.yaml @@ -51,7 +51,9 @@ repos: types: [text] # Every dependency on its ecosystem's newest STABLE release, exact-pinned - # so the question is mechanically answerable at all. + # so the question is mechanically answerable at all. That includes the + # native Proton build, pinned in proton-native.lock and checked against + # cachyos.db by the shared gate's pacman-repo adapter. # # `files:` is what keeps this cheap: a commit touching only source makes # ZERO network requests, and manifest commits mostly hit a 6h disk cache. @@ -71,7 +73,8 @@ repos: |package\.json|package-lock\.json|pnpm-lock\.yaml |Cargo\.(toml|lock)|go\.(mod|sum) |\.fvmrc|\.nvmrc|\.python-version - |\.github/workflows/.*\.ya?ml)$ + |\.github/workflows/.*\.ya?ml + |steam_installer/data/proton-native\.lock)$ - id: repo-contract name: repo contract (CLAUDE.md Commands + test_changed) diff --git a/_installer_import/README.md b/_installer_import/README.md index c8eeab9..e7b6bd2 100644 --- a/_installer_import/README.md +++ b/_installer_import/README.md @@ -8,9 +8,11 @@ a real install so **Play** works. ./run.sh BeamNG.drive ``` -That is the whole thing. `run.sh` installs its own dependencies (steamcmd via -`yay`, DepotDownloader from the SteamRE release, and a local `.venv`) on first -use, then works out what you meant. All four of these install the same game: +That is the whole thing. `run.sh` installs its own dependencies (`libibus` via +pacman, DepotDownloader from the SteamRE release, a local `.venv` and the native +Proton described below) on first use, then works out what you meant. It never +runs `steamcmd`: app info comes from Steam's own cache and the tool writes the +manifest itself. All four of these install the same game: ```bash ./run.sh 284160 # an AppID @@ -25,8 +27,49 @@ name shows a numbered picker, and in a non-interactive run it lists the candidates and exits 2 rather than guessing. Useful flags: `--dry-run` (resolve and report, change nothing), `--yes` -(no prompts), `--repair-ghosts` (see below), `--username` (remembered after the -first run, in `~/.config/steam-game-installer/config`; never a password). +(no prompts), `--repair-ghosts` (see below), `--valve-proton` (keep this game on +Valve's Proton for this run), `--map-installed` (see below), `--username` +(remembered after the first run, in `~/.config/steam-game-installer/config`; +never a password). + +## Games run on a FOSS Proton + +A plain Windows game is mapped to `proton-cachyos-native`, a Proton that +CachyOS builds from source against the host's own libraries. Steam then runs +it with no Steam Linux Runtime container, and Proton Experimental, Steam Linux +Runtime 4.0 and the Steamworks Redistributables are not needed while only such +games are installed (steam-backlog-enforcer removes them then). + +- **The tool.** `install.sh` downloads the exact build pinned in + `steam_installer/data/proton-native.lock` from the CachyOS mirror. It checks + the package's `.sig` against the CachyOS key pinned in + `steam_installer/data/cachyos_signing_key.asc` (fingerprint and sources in + `proton_pkg.py`), and its sha256 against the lock. Then it extracts only the + tool folder into `compatibilitytools.d/` and writes a `VERSION` stamp; the + package's `.INSTALL` is never run. A copy already on disk is adopted when + every file matches the package's `.MTREE`. An update is swapped in by rename, + only while Steam and every game are closed. The dependency-freshness gate + checks the lock against `cachyos.db`. +- **Which games.** Only a game proven safe: Windows-only (the app, every depot, + every launch option), no shared depots, no owned DLC depot, not in + AreWeAntiCheatYet's `games.json` (fetched every run, last good copy cached; + no list means nothing is mapped), and not in + `steam_installer/data/compat_overrides.json` (`{"": ""}` for a + game that breaks on the native build). Anything else keeps Valve's default, + so it runs exactly as before. The run prints every reason. +- **The mapping.** With Steam closed, the game gets a `CompatToolMapping` + entry in `config/config.vdf` (`"name" "proton-cachyos-native"`, priority + 250). The file is edited as text, and none of its other lines is ever logged. + An entry Valve chose (priority 100) is replaced; one you picked is kept. +- **Games already installed.** `./run.sh --map-installed` prints the decision + for every installed game and writes nothing; add `--apply` to close Steam and + write the safe ones. `--appids 730,1201270` decides any cached app instead. + +Proven live on 2026-11-22 with A Space for the Unbound (1201270). Not tested +yet, so they stay on Valve's default or are fixture-tested only: a game with +shared depots (its from-scratch manifest records `SharedDepots`), an owned DLC +depot, a native Linux game, an anti-cheat game, a global default mapping and a +package update. **Steam gets closed for you.** Steam owns `appmanifest_.acf` while it runs and rewrites it from its own in-memory state, so anything written @@ -76,7 +119,8 @@ password nobody was there to type. ## Ghost installs -`steamcmd +login anonymous +app_update ` has no licence for the app. +Older versions of this tool ran steamcmd. `steamcmd +login anonymous +app_update +` has no licence for the app. It does not say so. It transfers nothing, prints `Success! App '...' fully installed.`, exits 0, and writes a manifest claiming `StateFlags=4` with `SizeOnDisk=0` and an empty `InstalledDepots`. @@ -111,17 +155,21 @@ manifest verbatim before and after every write, the depot cache listing, expected-vs-actual depots, and whether Steam was running. Send `latest.log` when something goes wrong. `--verbose` mirrors all of it to the console. -Subprocess output is teed through a pty rather than captured, because steamcmd -only prompts for a password / Steam Guard code when it believes it is on a -terminal — with a plain pipe the prompt never appears and the run looks hung. +Subprocess output is teed through a pty rather than captured, because +depotdownloader only prompts for a password / Steam Guard code when it believes +it is on a terminal — with a plain pipe the prompt never appears and the run looks hung. ## How it works -1. `depotdownloader` pulls the content (fast path). +1. App info (name, installdir, depots, build) is read from Steam's own cache, + `appcache/appinfo.vdf`. An app missing from it fails with the fix: start + Steam once, let it sync, rerun. A DLC missing from it counts as not owned. +2. `depotdownloader` pulls the content (fast path). Then each owned DLC with depots of its own gets a run of its own. -2. `steamcmd app_update validate` reconciles Steam's own bookkeeping. -3. If that fails or leaves the manifest incomplete, the manifest is finished by - hand — but only when every precondition below holds. +3. The manifest is finished by hand, or written from scratch with the 15 keys + Steam accepted when none exists, but only when every precondition below + holds. Steam adds `LastOwner`, `UserConfig` and the rest itself. +4. The game is mapped to the native Proton when it is safe (see above). ## Fail-closed preconditions @@ -132,25 +180,23 @@ The tool refuses to mark an app installed, and exits non-zero, unless: (required = this app's depots for the chosen language and OS, excluding DLC, optional and shared depots — KCD2 lists 12 depots but an english/windows install only ever fetches 3); -- the manifest carries a real `buildid` matching the current public build; +- every required depot holds the public gid from Steam's cache, which proves + the build (a differing gid means the cache is stale: start Steam, let it + sync, rerun), or else the manifest carries a matching real `buildid`; - no Steam client is running (it is closed first, and restarted afterwards); - the manifest reads back as a real install *after* being written. This matters: an earlier version wrote `StateFlags=4` and reported success after -a steamcmd run that had done nothing at all. +a steamcmd run that had done nothing at all. A from-scratch manifest is built in +memory and only written once every check has passed. ## Failure modes this encodes Each of these cost a real debugging session; the tests pin them. -**`force_install_dir` is a library root, not an install directory.** Pointing it -at the game folder made steamcmd look for manifests in `/steamapps/`, find -none (`Loaded 0 apps from install folder`), create that stray directory, and then -fail the app with `Invalid platform` — it had no depot context for the app. -Dropped entirely: the app already lives in a registered library. - -**A Linux steamcmd needs `+@sSteamCmdForcePlatformType windows`** (before -`+login`) for a Windows-only game, or it resolves depots against the host OS. +**A wrong `installdir` makes Steam wipe the game.** Steam treated the correctly +downloaded folder as orphaned, deleted it and started a fresh download into the +folder its manifest named. The field is fixed first, as soon as Steam is closed. **Steam rejects an internally inconsistent manifest.** A file that says `StateFlags=4` while `BytesDownloaded` (22 GB) still differs from diff --git a/_installer_import/install.sh b/_installer_import/install.sh index 2ae395e..ae43f40 100755 --- a/_installer_import/install.sh +++ b/_installer_import/install.sh @@ -3,13 +3,20 @@ # ============================================================================ # Install everything this tool shells out to, then set up the dev environment. # -# steamcmd AUR package, via yay +# libibus the one runtime library of proton-cachyos-native's 63 +# package dependencies a plain Arch desktop may lack (pacman) # depotdownloader official self-contained release from SteamRE/DepotDownloader # .venv local virtualenv with the package installed editable +# native Proton proton-cachyos-native, pinned in +# steam_installer/data/proton-native.lock, signature- and +# sha256-checked, into Steam's compatibilitytools.d +# +# steamcmd is not needed any more: app info comes from Steam's own cache and +# the manifest is written by the tool itself. # # Idempotent: every step no-ops when its target is already in place, so run.sh -# can call it on every run. `yay` shells out to `sudo pacman`, so installing -# steamcmd may prompt for a password -- that is the one interactive step. +# can call it on every run. Installing libibus runs `sudo -n pacman`; when sudo +# would prompt, it stops and prints the one command to type. # ============================================================================ set -euo pipefail @@ -45,19 +52,29 @@ die() { exit 1 } -ensure_steamcmd() { - if command -v steamcmd >/dev/null 2>&1; then +ensure_libibus() { + if pacman -Q libibus >/dev/null 2>&1; then return 0 fi if [[ "$CHECK_ONLY" -eq 1 ]]; then - echo "missing: steamcmd -- run: yay -S steamcmd" >&2 + echo "missing: libibus -- run: sudo pacman -S --needed libibus" >&2 return 1 fi - command -v yay >/dev/null 2>&1 || - die "steamcmd is missing and yay is not installed. Install it, then: yay -S steamcmd" - echo "Installing steamcmd (yay may ask for your sudo password)..." - yay -S --needed --noconfirm --answerdiff None --answeredit None steamcmd - command -v steamcmd >/dev/null 2>&1 || die "steamcmd still not on PATH after install" + sudo -n pacman -S --needed --noconfirm libibus || + die "libibus is missing and sudo needs a password. Run: sudo pacman -S --needed libibus" +} + +# Never runs the package's .INSTALL; see steam_installer/proton_native.py. +ensure_native_proton() { + local python="${VENV}/bin/python" + if [[ "$CHECK_ONLY" -eq 1 ]]; then + if [[ -x "$python" ]] && ! "$python" -m steam_installer.proton_native --check; then + echo "missing: proton-cachyos-native at the pinned version -- run ./install.sh" >&2 + return 1 + fi + return 0 + fi + "$python" -m steam_installer.proton_native || die "native Proton install failed" } # Build from source only if the published binary can't be fetched. The upstream @@ -137,15 +154,17 @@ setup_dev_env() { main() { local missing=0 - ensure_steamcmd || missing=1 + ensure_libibus || missing=1 ensure_depotdownloader || missing=1 if [[ "$CHECK_ONLY" -eq 1 ]]; then + ensure_native_proton || missing=1 [[ "$missing" -eq 0 ]] && echo "All dependencies present." exit "$missing" fi setup_dev_env + ensure_native_proton echo "All dependencies present." : >"$STAMP" } diff --git a/_installer_import/pyproject.toml b/_installer_import/pyproject.toml index 2e02155..697f19a 100644 --- a/_installer_import/pyproject.toml +++ b/_installer_import/pyproject.toml @@ -1,11 +1,11 @@ [project] name = "steam-game-installer" version = "0.1.0" -description = "Fast Steam game installs: depotdownloader for speed, steamcmd for recognition." +description = "Fast Steam game installs: depotdownloader for speed, a hand-written manifest for recognition, a FOSS Proton to run them." requires-python = ">=3.14" # kuhy's shared "Created by kuhy" credit, shown at the end of --help. dependencies = [ - "kuhy-credit @ git+https://github.com/kuhyx/utils@kuhy-credit-v0.3.0#subdirectory=kuhy_credit", + "kuhy-credit @ git+https://github.com/kuhyx/utils@kuhy-credit-v0.3.2#subdirectory=kuhy_credit", ] [project.scripts] diff --git a/_installer_import/run.sh b/_installer_import/run.sh index 27ff431..8d3aa73 100755 --- a/_installer_import/run.sh +++ b/_installer_import/run.sh @@ -20,6 +20,7 @@ readonly SCRIPT_DIR readonly VENV_PY="${SCRIPT_DIR}/.venv/bin/python" readonly STAMP="${SCRIPT_DIR}/.venv/.deps-ok" readonly INSTALLER="${SCRIPT_DIR}/install.sh" +readonly PROTON_LOCK="${SCRIPT_DIR}/steam_installer/data/proton-native.lock" SKIP_DEPS="${SGI_SKIP_DEPS:-0}" DEPS_ONLY=0 @@ -52,7 +53,9 @@ deps_are_current() { [[ -f "$STAMP" ]] || return 1 [[ "$STAMP" -nt "$INSTALLER" ]] || return 1 [[ "$STAMP" -nt "${SCRIPT_DIR}/pyproject.toml" ]] || return 1 - "$VENV_PY" -c 'import steam_installer' >/dev/null 2>&1 + [[ "$STAMP" -nt "$PROTON_LOCK" ]] || return 1 + # A bumped pin or a deleted tool folder: the stamp alone would miss both. + "$VENV_PY" -m steam_installer.proton_native --check >/dev/null 2>&1 } main() { diff --git a/_installer_import/scripts/deploy.sh b/_installer_import/scripts/deploy.sh index a5a19a1..ed075f1 100755 --- a/_installer_import/scripts/deploy.sh +++ b/_installer_import/scripts/deploy.sh @@ -3,7 +3,7 @@ # ============================================================================ # Deploy origin/main to where this tool runs: kuhy's checkout # (~/src/steam-game-installer, used through ./run.sh). Refuses a dirty tree, -# fast-forwards it, reruns ./run.sh --deps-only (steamcmd, depotdownloader, +# fast-forwards it, reruns ./run.sh --deps-only (libibus, depotdownloader, the native Proton, # editable .venv), then starts the CLI as the health check. Nothing is # published elsewhere: no package, no service, no release. # Idempotent; non-zero unless the checkout is at origin/main and the CLI starts. diff --git a/_installer_import/steam_installer/anticheat.py b/_installer_import/steam_installer/anticheat.py new file mode 100644 index 0000000..dc4eb1a --- /dev/null +++ b/_installer_import/steam_installer/anticheat.py @@ -0,0 +1,57 @@ +"""Steam app ids that use anti-cheat, from AreWeAntiCheatYet's `games.json`. + +Fetched on every run; the last good copy is kept so a flaky network does not +change any decision. With neither a fetch nor a copy the answer is "unknown", +and an unknown list maps nothing. Every listed game counts, whatever its +status: kuhy's library has 54 such games (CS2, Garry's Mod, TF2, ...), and VAC +runs inside Valve's client, so those stay on Valve's own setup. +""" + +from __future__ import annotations + +import json +import os +import urllib.request +from pathlib import Path + +URL = ( + "https://raw.githubusercontent.com/AreWeAntiCheatYet/AreWeAntiCheatYet/" + "HEAD/games.json" +) +CACHE = Path.home() / ".cache" / "steam-game-installer" / "anticheat-games.json" + + +def _download() -> bytes: + with urllib.request.urlopen(URL, timeout=20) as response: + return response.read() + + +def _ids(raw: bytes | str) -> set[int]: + """Steam ids in a games.json; ValueError/TypeError when it is not one.""" + games = json.loads(raw) + if not isinstance(games, list): + raise TypeError("games.json is not a list") + ids: set[int] = set() + for game in games: + stores = game.get("storeIds") if isinstance(game, dict) else None + steam = stores.get("steam") if isinstance(stores, dict) else None + if isinstance(steam, str) and steam.isdigit(): + ids.add(int(steam)) + return ids + + +def steam_ids(cache: Path = CACHE) -> set[int] | None: + """Anti-cheat Steam app ids, or None when no list can be had.""" + try: + raw = _download() + ids = _ids(raw) + except (OSError, ValueError, TypeError): + try: + return _ids(cache.read_bytes()) + except (OSError, ValueError, TypeError): + return None + cache.parent.mkdir(parents=True, exist_ok=True) + tmp = cache.with_name(f"{cache.name}.tmp{os.getpid()}") + tmp.write_bytes(raw) + os.replace(tmp, cache) + return ids diff --git a/_installer_import/steam_installer/appinfo.py b/_installer_import/steam_installer/appinfo.py index c53e961..bd7f014 100644 --- a/_installer_import/steam_installer/appinfo.py +++ b/_installer_import/steam_installer/appinfo.py @@ -6,7 +6,6 @@ from dataclasses import dataclass, field from steam_installer import appinfo_cache - # depotdownloader is never given -branch, so it always fetches this one. PUBLIC_BRANCH = "public" diff --git a/_installer_import/steam_installer/cli.py b/_installer_import/steam_installer/cli.py index 8275ce0..8712ae8 100644 --- a/_installer_import/steam_installer/cli.py +++ b/_installer_import/steam_installer/cli.py @@ -13,6 +13,7 @@ from steam_installer import ( deps, flow, ghost, + map_installed, resolve, runlog, takeover, @@ -70,6 +71,7 @@ def _build_parser() -> argparse.ArgumentParser: "steamcmd leaves behind, and clear them away." ), ) + map_installed.add_arguments(parser) parser.add_argument( "--verbose", action="store_true", @@ -82,8 +84,11 @@ def main(argv: list[str] | None = None) -> int: parser = _build_parser() args = parser.parse_args(argv) - if not args.repair_ghosts and args.appid is None and args.target is None: - parser.error("provide an AppID, a Steam URL, a game name, or --repair-ghosts") + if not (args.repair_ghosts or args.map_installed) and not (args.appid or args.target): + parser.error( + "provide an AppID, a Steam URL, a game name, --repair-ghosts or --map-installed" + ) + map_installed.check_arguments(parser, args) literal = target.parse_literal(args.target) if args.target else None log = runlog.setup(args.appid or (literal[0] if literal else None), args.verbose) @@ -163,6 +168,8 @@ def _run(args: argparse.Namespace, log: runlog.RunLog) -> int: if args.repair_ghosts: return _repair_ghosts(log) + if args.map_installed: + return map_installed.run(args, log, find_steam_root()) try: deps.require() @@ -184,7 +191,8 @@ def _run(args: argparse.Namespace, log: runlog.RunLog) -> int: resolved.source, ) - libraries = list_libraries(find_steam_root()) + steam_root = find_steam_root() + libraries = list_libraries(steam_root) install_target = pick_install_target(libraries, metadata.installdir) library = next(lib for lib in libraries if lib.common == install_target.parent) report = ghost.inspect(library, metadata.appid) @@ -223,6 +231,7 @@ def _run(args: argparse.Namespace, log: runlog.RunLog) -> int: return flow.install( args=args, log=log, + steam_root=steam_root, library=library, metadata=metadata, install_target=install_target, diff --git a/_installer_import/steam_installer/compat_apply.py b/_installer_import/steam_installer/compat_apply.py new file mode 100644 index 0000000..9e271a2 --- /dev/null +++ b/_installer_import/steam_installer/compat_apply.py @@ -0,0 +1,74 @@ +"""Run the per-game decision and, when it says so, write the mapping. + +Shared by the install flow (right after the manifest is finished, while Steam +is still closed) and by `--map-installed`. The decision rules live in +:mod:`compat_decide`, the config.vdf edit in :mod:`compat_map`. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from steam_installer import anticheat, appinfo, compat_decide, compat_map, proton_pkg + +VALVE_PROTON = "--valve-proton" +TOOL_DIR = "compatibilitytools.d/proton-cachyos-native" + + +class Context: + """What every decision in one run shares: fetched once, not per game.""" + + def __init__(self, steam_root: Path, extra_overrides: dict[str, str] | None = None): + self.steam_root = steam_root + self.config = compat_map.config_path(steam_root) + self.anticheat = anticheat.steam_ids() + self.overrides = {**compat_decide.load_overrides(), **(extra_overrides or {})} + self.tool_ready = compat_decide.tool_ready( + steam_root / TOOL_DIR, proton_pkg.read_lock().version + ) + + def decide(self, appid: int, block: dict, owned_dlc: bool) -> compat_decide.Decision: + return compat_decide.decide( + appid, + block, + anticheat=self.anticheat, + overrides=self.overrides, + owned_dlc=owned_dlc, + tool_ready=self.tool_ready, + ) + + +def after_install( + steam_root: Path, + appid: int, + owned_dlc: bool, + valve_proton: bool, + logger: logging.Logger, +) -> None: + """Map a freshly installed game, or say why it keeps Valve's default. + + Never fails the install: a game left unmapped still runs on Valve's + Proton, exactly as before this tool knew about compat tools. + """ + extra = {str(appid): VALVE_PROTON} if valve_proton else {} + try: + context = Context(steam_root, extra) + decision = context.decide(appid, appinfo._fetch_app_block(appid), owned_dlc) + if not decision.map: + logger.info("Compat tool: %s", decision.describe()) + return + outcome = compat_map.map_app(context.config, appid) + except (appinfo.AppInfoError, compat_map.CompatMapError, OSError, ValueError) as exc: + logger.warning("Compat tool not set (%s); the game stays on Valve's default.", exc) + return + report(logger, appid, outcome) + + +def report(logger: logging.Logger, appid: int, outcome: compat_map.Outcome) -> None: + if outcome.action == "kept": + logger.info("Compat tool for %s kept: %s", appid, outcome.detail) + elif outcome.action == "already": + logger.info("Compat tool for %s: already mapped to %s.", appid, compat_map.TOOL) + else: + logger.info("Compat tool for %s: mapped to %s.", appid, compat_map.TOOL) diff --git a/_installer_import/steam_installer/compat_decide.py b/_installer_import/steam_installer/compat_decide.py new file mode 100644 index 0000000..d366c05 --- /dev/null +++ b/_installer_import/steam_installer/compat_decide.py @@ -0,0 +1,108 @@ +"""Decide per game whether Steam may run it on the native Proton. + +Only a game this can prove safe is mapped: Windows-only (the app, every depot +and every launch option), no shared depots, no owned DLC depot, not in the +anti-cheat list, not overridden. Anything else keeps Valve's default, so +Valve's tools come back while that game is installed -- every game stays +playable. When unsure, the answer is no. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +OVERRIDES = Path(__file__).parent / "data" / "compat_overrides.json" + + +@dataclass +class Decision: + appid: int + reasons: list[str] = field(default_factory=list) + + @property + def map(self) -> bool: + return not self.reasons + + def describe(self) -> str: + return "map" if self.map else f"Valve default: {', '.join(self.reasons)}" + + +def load_overrides(path: Path = OVERRIDES) -> dict[str, str]: + """`{appid: reason}` for games that break on the native build.""" + data = json.loads(path.read_text()) + if not isinstance(data, dict) or not all( + key.isdigit() and isinstance(reason, str) for key, reason in data.items() + ): + raise ValueError(f"{path.name}: compat_overrides must map app ids to reasons") + return data + + +def tool_ready(tool_dir: Path, pinned_version: str) -> bool: + """The tool is there and is the exact pinned, verified build.""" + stamp = tool_dir / "VERSION" + return ( + (tool_dir / "toolmanifest.vdf").is_file() + and stamp.is_file() + and stamp.read_text().strip() == pinned_version + ) + + +def _oses(value: object) -> set[str]: + return {os for os in value.split(",") if os} if isinstance(value, str) else set() + + +def _dicts(section: object) -> list[dict]: + if not isinstance(section, dict): + return [] + return [entry for entry in section.values() if isinstance(entry, dict)] + + +def _config_oslist(entry: dict) -> set[str]: + config = entry.get("config") + return _oses(config.get("oslist")) if isinstance(config, dict) else set() + + +def _platform_reasons(block: dict) -> list[str]: + common = block.get("common") + app_oses = _oses(common.get("oslist")) if isinstance(common, dict) else set() + depots = _dicts(block.get("depots")) + config = block.get("config") + launches = _dicts(config.get("launch")) if isinstance(config, dict) else [] + oses = app_oses.union(*(_config_oslist(e) for e in depots + launches)) + reasons = [] + if not app_oses: + reasons.append("no OS list") + if "linux" in oses: + reasons.append("native") + if oses - {"windows", "linux"}: + reasons.append("not Windows-only") + if any("depotfromapp" in d or "sharedinstall" in d for d in depots): + reasons.append("shared depots") + return reasons + + +def decide( + appid: int, + block: dict, + *, + anticheat: set[int] | None, + overrides: dict[str, str], + owned_dlc: bool, + tool_ready: bool, +) -> Decision: + """Every reason `appid` must keep Valve's default; none means map it.""" + reasons: list[str] = [] + if anticheat is None: + reasons.append("anti-cheat list unavailable") + elif appid in anticheat: + reasons.append("anti-cheat") + reasons += _platform_reasons(block) + if owned_dlc: + reasons.append("owned DLC") + if str(appid) in overrides: + reasons.append(f"override: {overrides[str(appid)]}") + if not tool_ready: + reasons.append("native Proton not installed") + return Decision(appid, reasons) diff --git a/_installer_import/steam_installer/compat_map.py b/_installer_import/steam_installer/compat_map.py new file mode 100644 index 0000000..f87c351 --- /dev/null +++ b/_installer_import/steam_installer/compat_map.py @@ -0,0 +1,208 @@ +"""Map one game to the native Proton in Steam's `config/config.vdf`. + +The entry goes under InstallConfigStore/Software/Valve/Steam/CompatToolMapping, +the same shape Steam writes for a per-game choice in its own UI: + + "" { "name" "proton-cachyos-native" "config" "" "priority" "250" } + +Steam itself records Valve's per-game defaults at priority 100. On 2026-11-22 +Steam honoured a 250 entry: `compat_log.txt` named the native tool as the only +tool. A global default was never tested, so only per-game entries are written. + +The file is edited as text, never round-tripped through a parser: every other +byte stays as Steam wrote it. It also holds private values (tokens, account +ids), so no line of it is ever logged or put in an error message -- errors +carry line numbers only. Steam rewrites the file from memory on exit, so a +write while it runs would be lost; it is refused. +""" + +from __future__ import annotations + +import os +import re +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +from steam_installer import recognize + +TOOL = "proton-cachyos-native" +PRIORITY = "250" +VALVE_PRIORITY = "100" +_STEAM = ("installconfigstore", "software", "valve", "steam") +_MAPPING = (*_STEAM, "compattoolmapping") +# A quoted string (escapes allowed, newlines too), a brace, or anything else. +_TOKEN = re.compile(r'"(?:[^"\\]|\\.)*"|[{}]|[^\s"{}]+') + + +class CompatMapError(Exception): + pass + + +@dataclass +class Outcome: + action: str # "mapped", "already" or "kept" + detail: str = "" + + +@dataclass +class _Entry: + open_line: int # the `{` after the appid key + close_line: int = -1 + values: dict[str, str] = field(default_factory=dict) + + +@dataclass +class _Layout: + lines: list[str] + steam_open: int = -1 + mapping_open: int = -1 + entry: _Entry | None = None + + +def config_path(steam_root: Path) -> Path: + return steam_root / "config" / "config.vdf" + + +def current(config: Path, appid: int) -> tuple[str, str] | None: + """The (name, priority) the game is mapped to, or None when unmapped.""" + entry = _scan(_read(config), str(appid)).entry + if entry is None: + return None + return entry.values.get("name", ""), entry.values.get("priority", "") + + +def map_app(config: Path, appid: int) -> Outcome: + """Point `appid` at the native Proton unless kuhy chose another tool.""" + pids = recognize.steam_client_pids() + if pids: + raise CompatMapError( + "Steam is running and would overwrite config.vdf on exit -- close it first" + ) + layout = _scan(_read(config), str(appid)) + lines = layout.lines + entry = layout.entry + if entry is not None: + name = entry.values.get("name", "") + priority = entry.values.get("priority", "") + if name == TOOL: + return Outcome("already") + if name and priority != VALVE_PRIORITY: + return Outcome("kept", f"already mapped to {name} (priority {priority})") + key_line = entry.open_line - 1 + if lines[key_line].strip() != f'"{appid}"' or lines[entry.close_line].strip() != "}": + raise CompatMapError(f"config.vdf line {key_line + 1}: unexpected layout") + indent = _indent(lines[entry.open_line]) + lines[key_line : entry.close_line + 1] = _entry(appid, indent) + _write(config, lines) + return Outcome("mapped", f"replaced Valve's choice {name or '(empty)'}") + if layout.mapping_open >= 0: + indent = _indent(lines[layout.mapping_open]) + "\t" + lines[layout.mapping_open + 1 : layout.mapping_open + 1] = _entry(appid, indent) + else: + one = _indent(lines[layout.steam_open]) + "\t" + block = [f'{one}"CompatToolMapping"\n', f"{one}{{\n"] + block += [*_entry(appid, one + "\t"), f"{one}}}\n"] + lines[layout.steam_open + 1 : layout.steam_open + 1] = block + _write(config, lines) + return Outcome("mapped") + + +def _read(config: Path) -> list[str]: + try: + return config.read_text(encoding="utf-8").splitlines(keepends=True) + except OSError as exc: + raise CompatMapError(f"cannot read {config}") from exc + + +def _tokens(lines: list[str]) -> list[tuple[str, int, bool]]: + """(text, line index, is_quoted) for every token; quoted values may span lines.""" + text = "".join(lines) + starts = [0] + for line in lines: + starts.append(starts[-1] + len(line)) + out: list[tuple[str, int, bool]] = [] + line_no = 0 + for match in _TOKEN.finditer(text): + while starts[line_no + 1] <= match.start(): + line_no += 1 + token = match.group(0) + if token[0] == '"': + out.append((token[1:-1], line_no, True)) + elif token in "{}": + out.append((token, line_no, False)) + else: + raise CompatMapError(f"config.vdf line {line_no + 1}: unexpected layout") + return out + + +def _scan(lines: list[str], appid: str) -> _Layout: + """Find the Steam block, its mapping block and the app's entry.""" + layout = _Layout(lines) + stack: list[str] = [] + pending: str | None = None + for text, index, quoted in _tokens(lines): + in_entry = layout.entry is not None and tuple(stack) == (*_MAPPING, appid) + if quoted: + if pending is None: + pending = text + continue + if in_entry: + layout.entry.values[pending.lower()] = text + pending = None + elif text == "{": + if pending is None: + raise CompatMapError(f"config.vdf line {index + 1}: block without a key") + stack.append(pending.lower()) + pending = None + _note_open(layout, tuple(stack), appid, index) + else: + if not stack or pending is not None: + raise CompatMapError(f"config.vdf line {index + 1}: unbalanced brace") + if in_entry: + layout.entry.close_line = index + stack.pop() + if layout.steam_open < 0: + raise CompatMapError("config.vdf has no InstallConfigStore/Software/Valve/Steam") + return layout + + +def _note_open(layout: _Layout, path: tuple[str, ...], appid: str, index: int) -> None: + if path in (_STEAM, _MAPPING, (*_MAPPING, appid)) and layout.lines[index].strip() != "{": + raise CompatMapError(f"config.vdf line {index + 1}: unexpected layout") + if path == _STEAM: + layout.steam_open = index + elif path == _MAPPING: + layout.mapping_open = index + elif path == (*_MAPPING, appid): + layout.entry = _Entry(open_line=index) + + +def _indent(line: str) -> str: + return line[: len(line) - len(line.lstrip())] + + +def _entry(appid: int, indent: str) -> list[str]: + inner = indent + "\t" + return [ + f'{indent}"{appid}"\n', + f"{indent}{{\n", + f'{inner}"name"\t\t"{TOOL}"\n', + f'{inner}"config"\t\t""\n', + f'{inner}"priority"\t\t"{PRIORITY}"\n', + f"{indent}}}\n", + ] + + +def _write(config: Path, lines: list[str]) -> None: + """Replace the file atomically, keeping its permissions.""" + mode = config.stat().st_mode & 0o777 + fd, tmp = tempfile.mkstemp(dir=config.parent, prefix=".config.vdf.") + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write("".join(lines)) + os.chmod(tmp, mode) + os.replace(tmp, config) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise diff --git a/_installer_import/steam_installer/config.py b/_installer_import/steam_installer/config.py index a86be4d..03b6233 100644 --- a/_installer_import/steam_installer/config.py +++ b/_installer_import/steam_installer/config.py @@ -1,9 +1,8 @@ """Remember the Steam username so every run doesn't ask for it again. Only the username is ever persisted. Passwords and Steam Guard codes stay -where they belong: with steamcmd's own cached login token (prompted through -the pty in `runlog.run_logged`) and depotdownloader's `-remember-password`. -Writing either of those to a dotfile would be a downgrade in security over +where they belong: with depotdownloader's `-remember-password` login token +(prompted through the pty in `runlog.run_logged`). Writing them to a dotfile would be a downgrade in security over what Steam's own tooling already does. """ diff --git a/_installer_import/steam_installer/data/cachyos_signing_key.asc b/_installer_import/steam_installer/data/cachyos_signing_key.asc new file mode 100644 index 0000000..90b0e88 --- /dev/null +++ b/_installer_import/steam_installer/data/cachyos_signing_key.asc @@ -0,0 +1,53 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: Hostname: +Version: Hockeypuck 2.2 + +xsDNBGERxNwBDAD1aEVN8rdWBlV2sOwkiW1ddO44xKjdumwuGhoXP1pUzK4SkNUb +IrQ1cenckdxzZsS8cbXTvEteC4K8tTYxobZpgzAPhKZNECsyCyDf2DEYv7bS75gy +sdl0bh+mOXPjvOe9IK3eFPQWWFd0erYyf9SHjlZ0utOg3qZ2JuVupGOH8W3taKth +ijR98RzUqP5rPoMPKcrOdBUiIREHx0y6SwdyX0c2XKm52T3cP8O0MPsvZLbRwGU0 +DTyr7knzuGhR9tLxsy4PqKz3tSUEGM7PhFcpEoRFvV/4Al8/OpRaPj3zRn6RbL1D +9ZAH9OblgHkQlK6MPnLOygoaAvjreMHK3ykjlRugUtnRBmQHhH+o3iXb8Ep5GXjZ +XJZazy5ZSTbuJ8+DcjuC/G459ePnQ658od2Sxz5OBRmK3a1taAtgszHio1EHb8Db +OZ+Za5E46VOHM/8sl8toGbK4JOqpgdQ3YqIH4R915WqCYnMvfTQbCiHLYzWHrgAA +awBu3kaV2MagPnkAEQEAAc0bQ2FjaHlPUyA8YWRtaW5AY2FjaHlvcy5vcmc+wsEO +BBMBCAA4AhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAFiEEiC3P5I4gUdSOJWKr +87YHSI2zWkcFAmTUdqcACgkQ87YHSI2zWkc2ygwAknD+O2LQeHp1tEYwplF9Rthq +uPN64IxC/8nAc+L92fSOkVJKUfRh0PJcCCZgpTBAa7UV++GsT9SZ8jRtUPDfAy/R +0TSwKkEC6b1291xNdV3O83CtkSJRTP39nVCefXrBUpIwmkQJuytBF1qypTuw4jom +orcbmE40O7uNDe0FpZOJkUxT1sVJqj+y2h4mJsDuEAYKVQa4809yDwlQOYlLg6i3 +wEBut7PPh8bEFWycNElKseh3dpQfvFVWMcUBafpQq7Q9zVFDW3nCUkFBMErcLb0y +VARQizrO3GqmWSU0zmgs8ruc3HnYjIfLIU68JRDsLdFgzIstEasnOskdUYywdvYw +BeGpCQX2ZFcTosLrZc1SRAk/XKioVXeHA92dcQacBClzKCmbat0CcNng5phu9fom +xGbNijvV5NWmQITw7j2sco0nLtfx2MiyRlR8s6pgrZ/VgfGq5K2gM306MgshgyMW +oIELm7aUTkbY+QH4v0y+U/KUNeHetDZhsMLwZq1DwsEUBBMBCAA+FiEEiC3P5I4g +UdSOJWKr87YHSI2zWkcFAmERxNwCGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwEC +HgECF4AACgkQ87YHSI2zWkcB8QwA5+lumQa7SItzfd5v7HJhDU9IEE6gvxhs9wPC +UxcZmbCVj4AoP0yLXshbKpbwfpXQN+CmlnNr+U9TmUe0q6f6L7qdqIAfDGOy2BOX +Xn/x1h5u7jeppUYpSiB4oZW2A6zHuWp1RyGSiPFO0W+k5awA8nBNnZpnCdZIjT2R +OWCpSOpGgR8V35joq/tUxje/A1W7oaHuWnuSIz8nEqHQc9xTXbU4tFab24cxRxV+ +3U1wJ10jC0VoBPX2B7KhutgEIqwq9V2XOfRUAKO2u5IcKKwYFhjHa4O951rm+YoV +L2fj3BNeQwwwufofEHat4GhVXhvw+oZli3PMZ/9PZBf40/T5I4m+t+4nkpyyl3ng +Z8iW55gGKkcjgEmGBurmB+KuRbSBG6gQrYIfVvil+QFRVFh9yQJNKX7lzZpBv6M2 +tc1kC8N0jV2pTjlTwjDhn5xm3qC00FHLuvp6kNNDpnkYxYhVYR5PVLE1TOyPqL4L +J3I93lNfpef8p6jjBEJXi6C8PL+mzsDNBGERxNwBDAC1RE2JFOjAMzqg3f5JsTa5 +1WgApe2zlS/QjD+qr2LbC0CvQK3WO/MseqnEBfnXlUrL+tCzQXG79+kJk5xHsdaI +bblTqdX3xNbT0/+sJkLsUWbfLh1lTgs+NnltsCtg1cIKm/eOBEWyVfnrkKI+uTlQ +DDGcHW9XGaxLaIMi8FgaitMtrvH3AdajEpvpvdAAUbeIUL/hFs6VTSs74xJRxy0J +1eZYlZXo2YQ8EuZCAtyaQuvoDKijybxqj+FKPRiL920UI2emBYL0cwMRq1XylZST +7xXEzrEVgQ+xEUTBOUgmbRmIJVsSE/pDy7127dwPVpTglWiwfqntgdujrl6HM2b0 +a6lV5/ZoeYL5ucu6bku0ewbkoEKhqTe1JSVYWbStW5hwGFBxCFxGpKQMMsDltmst +8Z0Ic1Bhes00gd7LAC0AK8OkWFs4sIbHinNBVXQH/5MINF2ngm9tP9kym17Nr5Jt +erbxOOyNYl+ZOq6HFsrMXMZ1Cx+d4diqK2GnSeCZIvMAEQEAAcLA/AQYAQgAJhYh +BIgtz+SOIFHUjiViq/O2B0iNs1pHBQJhEcTcAhsMBQkDwmcAAAoJEPO2B0iNs1pH +mFMMAOuyEJ93J4X5H+Jnf8/pXg2PFVvp0se7weCFha1d9umGSTrQEjivm55z1X4R ++Jt6D/cL3wN1WnayyNEtVzTD1CiuMDNO4a+qyBmE3/tJRWyIXoZ/wdNu38rRNO/S +dNbDWj0dCl6HCokLKSlfpP7996zymNeVX7yCw7T56jb2mTILTszyE0xYPVAKb8cR +Zt9gUTivxQ3SPw5xVe1OXcF0Dkwx2EAAm7Alo6DMCZlqTii8Xc6KYcaSRHNdlmkk +W3RcTMkNja5mRFhzygklh9ViXqQs+9gZB+VHDqB2iNKbHJwO40aLRo/bta1h9P0H +DD8q8mZWkZpGBlzul0x5UMedCeYudk9tOMwFCKRrhZeAyzMRn/+FXUw7oFiWX6RX +9ujq0O+Yna6WsWe+YN/vmT0KloTzRb0yqeDc37zc850+woYy+mNpV2+Tq6hBDajk ++BohvFyI7T4vvs45e+GwN2fTS6oqVsc9TfO7Q8JTi5y9UqByEbNCbqY/IeIwaapR +IqDYsg== +=N+41 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/_installer_import/steam_installer/data/compat_overrides.json b/_installer_import/steam_installer/data/compat_overrides.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/_installer_import/steam_installer/data/compat_overrides.json @@ -0,0 +1 @@ +{} diff --git a/_installer_import/steam_installer/data/proton-native.lock b/_installer_import/steam_installer/data/proton-native.lock new file mode 100644 index 0000000..9c7a349 --- /dev/null +++ b/_installer_import/steam_installer/data/proton-native.lock @@ -0,0 +1,7 @@ +# pacman-repo lock: the exact proton-cachyos-native build steam-game-installer +# installs. The dependency-freshness gate (kuhyx/utils, pacman-repo adapter) +# compares it with cachyos.db; bump version and sha256 together. +name=proton-cachyos-native +repo=https://mirror.cachyos.org/repo/x86_64/cachyos/cachyos.db +version=1:11.0.20261015-1 +sha256=25bf302f90e8e01118555b997b873c3cd06a33b5647e79d049a59e29794b43e0 diff --git a/_installer_import/steam_installer/finish_check.py b/_installer_import/steam_installer/finish_check.py index a455c02..fbcb644 100644 --- a/_installer_import/steam_installer/finish_check.py +++ b/_installer_import/steam_installer/finish_check.py @@ -87,7 +87,7 @@ def check_ready_to_finish( if not (buildid and buildid != "0") and not (target and target != "0"): problems.append( "manifest has no usable buildid (both buildid and TargetBuildID are 0) " - "-- steamcmd never recorded a real build" + "-- no real build was ever recorded" ) resolved = buildid if buildid and buildid != "0" else target if expected_build_id is None: diff --git a/_installer_import/steam_installer/flow.py b/_installer_import/steam_installer/flow.py index fb27e8d..848b6fd 100644 --- a/_installer_import/steam_installer/flow.py +++ b/_installer_import/steam_installer/flow.py @@ -10,6 +10,7 @@ from pathlib import Path from steam_installer import ( appinfo, + compat_apply, depotdl, dlc, manifest_write, @@ -50,6 +51,7 @@ def install( *, args, log: runlog.RunLog, + steam_root: Path, library: SteamLibrary, metadata: appinfo.AppMetadata, install_target: Path, @@ -64,9 +66,22 @@ def install( if steam_was_running is None: return 1 try: - return _download_and_recognize( + returncode = _download_and_recognize( args, log, library, metadata, install_target, staged, username, language ) + if returncode == 0: + # Still inside the window where Steam is closed: config.vdf written + # under a running client is lost when it exits. + cached = recognize.discover_cached_depot_manifests(install_target) + dlc_depots = appinfo.get_depot_plan(metadata.appid, language).dlc + compat_apply.after_install( + steam_root, + metadata.appid, + owned_dlc=bool(set(dlc_depots) & set(cached)), + valve_proton=args.valve_proton, + logger=logger, + ) + return returncode finally: if steam_was_running: steamctl.start(logger) diff --git a/_installer_import/steam_installer/map_installed.py b/_installer_import/steam_installer/map_installed.py new file mode 100644 index 0000000..8ed5731 --- /dev/null +++ b/_installer_import/steam_installer/map_installed.py @@ -0,0 +1,125 @@ +"""`--map-installed`: run the compat decision over every installed game. + + ./run.sh --map-installed # dry run: print, write nothing + ./run.sh --map-installed --apply # close Steam, map the safe ones + ./run.sh --map-installed --appids 730,10 # decide any cached app (read-only) + +The dry run only reads, so it works with Steam open. `--apply` closes Steam +first (it rewrites config.vdf on exit) and restarts it afterwards. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from steam_installer import ( + appinfo, + compat_apply, + compat_map, + flow, + recognize, + runlog, + steamctl, +) +from steam_installer.library import SteamLibrary, list_libraries + +# Steam's own compat tools and runtimes are apps too, but not games. +TOOL_TYPES = {"tool", "config"} + + +def add_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--valve-proton", + action="store_true", + help="Keep this game on Valve's default Proton for this run.", + ) + parser.add_argument( + "--map-installed", + action="store_true", + help="Decide every installed game's compat tool. Dry run unless --apply.", + ) + parser.add_argument( + "--apply", action="store_true", help="With --map-installed: write the mappings." + ) + parser.add_argument( + "--appids", + default=None, + help="With --map-installed: comma-separated app ids to decide instead.", + ) + + +def check_arguments(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: + if args.map_installed and args.valve_proton: + parser.error("--valve-proton goes with an install, not --map-installed") + + +def _installed(libraries: list[SteamLibrary]) -> dict[int, dict]: + """appid -> AppState for every manifest in every library.""" + found: dict[int, dict] = {} + for library in libraries: + for manifest in sorted(library.steamapps.glob("appmanifest_*.acf")): + appid = manifest.stem.removeprefix("appmanifest_") + if appid.isdigit(): + found[int(appid)] = recognize.read_app_state(manifest) + return found + + +def _owned_dlc(app_state: dict) -> bool: + depots = app_state.get("InstalledDepots") + entries = depots.values() if isinstance(depots, dict) else [] + return any(isinstance(entry, dict) and "dlcappid" in entry for entry in entries) + + +def _is_tool(block: dict) -> bool: + common = block.get("common") + kind = common.get("type", "") if isinstance(common, dict) else "" + return isinstance(kind, str) and kind.lower() in TOOL_TYPES + + +def run(args: argparse.Namespace, log: runlog.RunLog, steam_root: Path) -> int: + logger = log.logger + installed = _installed(list_libraries(steam_root)) + if args.appids: + appids = [int(x) for x in args.appids.split(",") if x.strip().isdigit()] + else: + appids = sorted(installed) + context = compat_apply.Context(steam_root) + blocks = appinfo._cached_blocks(appids) + to_map: list[int] = [] + for appid in appids: + block = blocks.get(appid) + if block is None: + logger.info("%s: not in Steam's app cache -- skipped", appid) + continue + if _is_tool(block) and not args.appids: + continue + name = block.get("common", {}).get("name", "?") + decision = context.decide(appid, block, _owned_dlc(installed.get(appid, {}))) + line = f"{appid} {name}: {decision.describe()}" + if decision.map: + to_map.append(appid) + current = compat_map.current(context.config, appid) + if current and current[0] == compat_map.TOOL: + line += " -- already mapped" + logger.info("%s", line) + if not args.apply: + logger.info("Dry run: nothing written. Rerun with --apply to write.") + return 0 + return _apply(args, logger, context.config, to_map) + + +def _apply(args, logger, config, to_map: list[int]) -> int: + was_running = flow._prepare_steam(args, logger) + if was_running is None: + return 1 + try: + for appid in to_map: + compat_apply.report(logger, appid, compat_map.map_app(config, appid)) + except compat_map.CompatMapError as exc: + logger.error("Mapping stopped: %s", exc) + return 1 + finally: + if was_running: + steamctl.start(logger) + return 0 diff --git a/_installer_import/steam_installer/proton_native.py b/_installer_import/steam_installer/proton_native.py new file mode 100644 index 0000000..c8e42d1 --- /dev/null +++ b/_installer_import/steam_installer/proton_native.py @@ -0,0 +1,173 @@ +"""Install the pinned proton-cachyos-native build into Steam's compatibilitytools.d. + + python -m steam_installer.proton_native # what install.sh runs + python -m steam_installer.proton_native --check # exit 0 when in place + +The package comes straight from the CachyOS mirror: the `proton-cachyos-native` +entry of `cachyos.db`, its signature and sha256 checked (see :mod:`proton_pkg`), +and only `usr/share/steam/compatibilitytools.d/proton-cachyos-native/` taken +out of it. Its `.INSTALL` is never run. It is a WoW64 build that needs no +lib32 packages, and unlike the `-slr` variant its toolmanifest requires no +Steam Linux Runtime -- which is the point. + +A `VERSION` stamp equal to the lock means there is nothing to do and no +network is touched. Otherwise the new build is extracted next to the old one +and swapped in by rename -- never while Steam or a game from the tool runs, +since a game would lose its files mid-run. A copy already on disk with no +stamp is adopted as is when every file matches the package's `.MTREE`. + +`SGI_PROTON_REPO` points the fetch at another copy of the repo db (a local +test mirror); the pinned key and sha256 still decide what is accepted. +""" + +from __future__ import annotations + +import dataclasses +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +from steam_installer import proton_pkg, recognize +from steam_installer.library import find_steam_root +from steam_installer.proton_pkg import FINGERPRINT, KEY, Lock, PkgError + +TOOL_NAME = "proton-cachyos-native" + + +def in_use(tool: Path) -> list[int]: + """Steam client pids plus every process running from the tool folder.""" + pids = list(recognize.steam_client_pids()) + root = tool.resolve() + for entry in recognize.PROC_DIR.iterdir(): + if not entry.name.isdigit(): + continue + try: + exe = (entry / "exe").resolve(strict=True) + except OSError: + continue + if exe.is_relative_to(root): + pids.append(int(entry.name)) + return pids + + +def _download(url: str, dest: Path) -> None: + with urllib.request.urlopen(url, timeout=60) as response, dest.open("wb") as out: + shutil.copyfileobj(response, out, 1 << 20) + + +def _fetch_verified(lock: Lock, work: Path, key: Path, fingerprint: str) -> Path: + """Download the pinned package and prove it, or raise PkgError.""" + db = work / "repo.db" + _download(lock.repo, db) + entry = proton_pkg.find_in_db(db, lock.name) + if entry.version != lock.version: + raise PkgError( + f"the repo serves {lock.name} {entry.version} but the lock pins " + f"{lock.version}: bump steam_installer/data/proton-native.lock " + "(version and sha256)" + ) + if entry.sha256 != lock.sha256: + raise PkgError("the repo's sha256 for the pinned version differs from the lock") + package = work / entry.filename + url = proton_pkg.package_url(lock, entry.filename) + _download(url, package) + sig = work / f"{entry.filename}.sig" + _download(f"{url}.sig", sig) + home = work / "gnupg" + home.mkdir(mode=0o700) + try: + proton_pkg.verify_signature(package, sig, key, fingerprint, home) + finally: + subprocess.run( + ["gpgconf", "--homedir", str(home), "--kill", "all"], + check=False, capture_output=True, + ) + if proton_pkg.sha256_file(package) != lock.sha256: + raise PkgError("the downloaded package's sha256 differs from the pinned sha256") + return package + + +def _stamp(tool: Path, version: str) -> None: + (tool / "VERSION").write_text(version + "\n") + + +def _swap(package: Path, tools_dir: Path, tool: Path, version: str) -> None: + new = tools_dir / f".{TOOL_NAME}.new" + old = tools_dir / f".{TOOL_NAME}.old" + for leftover in (new, old): + shutil.rmtree(leftover, ignore_errors=True) + new.mkdir() + proton_pkg.extract(package, new) + if tool.exists(): + tool.rename(old) + new.rename(tool) + shutil.rmtree(old, ignore_errors=True) + _stamp(tool, version) + + +def ensure( + tools_dir: Path, lock: Lock, *, key: Path = KEY, fingerprint: str = FINGERPRINT +) -> str: + """Make `tools_dir/proton-cachyos-native` the pinned build; return what happened.""" + tool = tools_dir / TOOL_NAME + stamp = tool / "VERSION" + current = stamp.read_text().strip() if stamp.is_file() else None + if current == lock.version: + return f"current: {lock.version}" + tools_dir.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(dir=tools_dir, prefix=".proton-download.")) + try: + package = _fetch_verified(lock, work, key, fingerprint) + checks = f"signature by {fingerprint} and sha256 both passed" + if not tool.exists(): + verb = f"installed: {lock.version}" + elif current is not None: + verb = f"updated: {current} -> {lock.version}" + else: + entries = proton_pkg.read_mtree(package) + mismatch = proton_pkg.first_mismatch(tool, entries) + if mismatch is None: + _stamp(tool, lock.version) + count = sum(entry.kind != "dir" for entry in entries.values()) + return f"adopted: all {count} files match .MTREE ({checks})" + verb = f"replaced: mismatch in {mismatch}" + busy = in_use(tool) if tool.exists() else [] + if busy: + return ( + f"deferred: Steam or a game is running (pid {', '.join(map(str, busy))}); " + "rerun with Steam closed" + ) + _swap(package, tools_dir, tool, lock.version) + return f"{verb} ({checks})" + finally: + shutil.rmtree(work, ignore_errors=True) + + +def main(argv: list[str] | None = None) -> int: + """Install the pinned build; with `--check`, only say whether it is in place.""" + lock = proton_pkg.read_lock() + if os.environ.get("SGI_PROTON_REPO"): + lock = dataclasses.replace(lock, repo=os.environ["SGI_PROTON_REPO"]) + try: + tools_dir = find_steam_root() / "compatibilitytools.d" + except FileNotFoundError: + print("native Proton skipped: no Steam installation") + return 0 + if argv and argv[0] == "--check": + stamp = tools_dir / TOOL_NAME / "VERSION" + return 0 if stamp.is_file() and stamp.read_text().strip() == lock.version else 1 + try: + status = ensure(tools_dir, lock) + except (PkgError, OSError) as exc: + print(f"error: native Proton: {exc}", file=sys.stderr) + return 1 + print(f"native Proton {status}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/_installer_import/steam_installer/proton_pkg.py b/_installer_import/steam_installer/proton_pkg.py new file mode 100644 index 0000000..74ab6e8 --- /dev/null +++ b/_installer_import/steam_installer/proton_pkg.py @@ -0,0 +1,208 @@ +"""The CachyOS package formats and the two checks every download must pass. + +The repo database is not signed, so nothing in it is trusted on its own: +- the package's detached `.sig` must verify against the CachyOS key pinned in + `data/cachyos_signing_key.asc`, by primary-key fingerprint, in a throwaway + gpg home that holds that key and nothing else; +- the package's sha256 must equal both the db entry and the pinned lock. +Either failing stops the install before anything is extracted. + +Pinned key: 882DCFE48E2051D48E2562ABF3B607488DB35A47 ("CachyOS +"). The .sig names its own issuer, so it could not vouch +for the pin. Taken on 2026-11-22 from two other sources instead: +https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x882DCFE48E2051D48E2562ABF3B607488DB35A47 +(the key file) and the long id F3B607488DB35A47 that CachyOS's own installer +imports, https://github.com/CachyOS/cachyos-repo-add-script/blob/5b27dc1eff276e104077cfaf75798b4c290eaef4/cachyos-repo.sh#L156 +""" + +from __future__ import annotations + +import gzip +import hashlib +import re +import subprocess +import tarfile +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import quote + +DATA = Path(__file__).parent / "data" +LOCK = DATA / "proton-native.lock" +KEY = DATA / "cachyos_signing_key.asc" +FINGERPRINT = "882DCFE48E2051D48E2562ABF3B607488DB35A47" +PREFIX = "usr/share/steam/compatibilitytools.d/proton-cachyos-native/" + + +class PkgError(Exception): + pass + + +@dataclass(frozen=True) +class Lock: + name: str + repo: str # URL of the .db + version: str + sha256: str + + +@dataclass(frozen=True) +class DbEntry: + filename: str + version: str + sha256: str + + +@dataclass(frozen=True) +class MtreeEntry: + kind: str # "file", "link" or "dir" + sha256: str = "" + link: str = "" + + +def read_lock(path: Path = LOCK) -> Lock: + fields = {} + for line in path.read_text().splitlines(): + if line.strip() and not line.lstrip().startswith("#"): + key, _, value = line.partition("=") + fields[key.strip()] = value.strip() + try: + return Lock(fields["name"], fields["repo"], fields["version"], fields["sha256"]) + except KeyError as exc: + raise PkgError(f"{path.name} has no {exc.args[0]}=") from exc + + +def package_url(lock: Lock, filename: str) -> str: + """The package sits next to the db; the epoch colon must be escaped.""" + return f"{lock.repo.rsplit('/', 1)[0]}/{quote(filename)}" + + +def find_in_db(db: Path, name: str) -> DbEntry: + """The db entry whose %NAME% is exactly `name` (not `-v3`).""" + with tarfile.open(db, "r:*") as tar: + for member in tar: + if not member.name.endswith("/desc"): + continue + fields = _desc(tar.extractfile(member).read().decode()) + if fields.get("NAME") == name: + return DbEntry(fields["FILENAME"], fields["VERSION"], fields["SHA256SUM"]) + raise PkgError(f"the repo database lists no {name}") + + +def _desc(text: str) -> dict[str, str]: + fields: dict[str, str] = {} + for block in text.strip().split("\n\n"): + head, _, value = block.partition("\n") + fields[head.strip("%")] = value.strip() + return fields + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _gpg(home: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["gpg", "--homedir", str(home), "--batch", "--no-tty", *args], + capture_output=True, text=True, check=False, timeout=60, + ) + + +def key_fingerprints(key: Path, home: Path) -> list[str]: + """Primary-key fingerprints in `key`, imported into the empty `home`.""" + _gpg(home, "--import", str(key)) + listing = _gpg(home, "--with-colons", "--fingerprint").stdout.splitlines() + out, after_pub = [], False + for line in listing: + if line.startswith("pub"): + after_pub = True + elif line.startswith("fpr") and after_pub: + out.append(line.split(":")[9]) + after_pub = False + return out + + +def verify_signature(package: Path, sig: Path, key: Path, fingerprint: str, home: Path) -> None: + """Fail unless `sig` is a good signature by the pinned primary key.""" + if fingerprint not in key_fingerprints(key, home): + raise PkgError("the pinned key file does not hold the pinned fingerprint") + result = _gpg(home, "--status-fd", "1", "--verify", str(sig), str(package)) + signers = [ + line.split()[-1] + for line in result.stdout.splitlines() + if line.startswith("[GNUPG:] VALIDSIG ") + ] + if result.returncode != 0 or signers != [fingerprint]: + raise PkgError(f"package signature does not verify against {fingerprint}") + + +def _unescape(path: str) -> str: + return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), path) + + +def read_mtree(package: Path) -> dict[str, MtreeEntry]: + """The package's own file list for the tool folder, relative to it.""" + with tarfile.open(package, "r:zst") as tar: + for member in tar: + if member.name == ".MTREE": + text = gzip.decompress(tar.extractfile(member).read()).decode() + break + else: + raise PkgError("package has no .MTREE") + entries: dict[str, MtreeEntry] = {} + defaults: dict[str, str] = {} + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + path, *pairs = line.split(" ") + values = dict(pair.partition("=")[::2] for pair in pairs) + if path == "/set": + defaults.update(values) + continue + path = _unescape(path.removeprefix("./")) + if not path.startswith(PREFIX) or path == PREFIX: + continue + values = {**defaults, **values} + entries[path[len(PREFIX):]] = MtreeEntry( + values.get("type", "file"), values.get("sha256digest", ""), values.get("link", "") + ) + return entries + + +def first_mismatch(folder: Path, entries: dict[str, MtreeEntry]) -> str | None: + """The first path where `folder` differs from the package, or None. + + Files the package does not list count as a difference (a stray + user_settings.py changes how Proton runs), except Python's own + `__pycache__`, which running protonfixes creates. + """ + for rel, entry in sorted(entries.items()): + path = folder / rel + if entry.kind == "link": + if not path.is_symlink() or str(path.readlink()) != entry.link: + return rel + elif entry.kind == "file" and ( + path.is_symlink() or not path.is_file() or sha256_file(path) != entry.sha256 + ): + return rel + for path in sorted(folder.rglob("*")): + if path.is_dir() and not path.is_symlink(): + continue + rel = str(path.relative_to(folder)) + if rel not in entries and "__pycache__" not in path.parts and rel != "VERSION": + return rel + return None + + +def extract(package: Path, dest: Path) -> None: + """Unpack only the tool folder into `dest`; scripts are never run.""" + with tarfile.open(package, "r:zst") as tar: + members = [] + for member in tar.getmembers(): + if member.name.startswith(PREFIX) and member.name != PREFIX: + member.name = member.name[len(PREFIX):] + members.append(member) + tar.extractall(dest, members=members, filter="data") diff --git a/_installer_import/steam_installer/recognize.py b/_installer_import/steam_installer/recognize.py index 25f6df6..a2a9a51 100644 --- a/_installer_import/steam_installer/recognize.py +++ b/_installer_import/steam_installer/recognize.py @@ -1,7 +1,7 @@ """Read-only questions about whether Steam will see an install as real. -Split on 2026-09-25 (250-line cap): steamcmd's validate run now lives in -:mod:`steamcmd_validate`, every manifest write in :mod:`manifest_write`, and +Split on 2026-09-25 (250-line cap): every manifest write lives in +:mod:`manifest_write`, and the preconditions for writing in :mod:`finish_check`. What stays here is the shared vocabulary they all read: depotdownloader's cache, Steam's staging tree and client process, and whether a manifest's bookkeeping says "done". diff --git a/_installer_import/steam_installer/runlog.py b/_installer_import/steam_installer/runlog.py index d3741de..ebcd38d 100644 --- a/_installer_import/steam_installer/runlog.py +++ b/_installer_import/steam_installer/runlog.py @@ -8,10 +8,10 @@ log records the manifest *verbatim, before and after* every write, alongside the full output of every subprocess. Subprocess output is teed through a pty rather than captured with -`capture_output=True`. steamcmd prompts for a password (and possibly a Steam -Guard code) on stdin; prompt text written to a captured/buffered pipe never +`capture_output=True`. depotdownloader prompts for a password (and possibly a +Steam Guard code) on stdin; prompt text written to a captured/buffered pipe never reaches the terminal, so the process looks hung with no visible request for -input. A pty keeps steamcmd in line-buffered interactive mode and lets us +input. A pty keeps it in line-buffered interactive mode and lets us mirror every byte to both the terminal and the log. """ @@ -110,7 +110,7 @@ def run_logged( """Run `cmd` on a pty, mirroring output to both the terminal and the log. Returns (returncode, combined_output). stdin stays attached to the real - terminal so interactive prompts (steamcmd's password / Steam Guard) work + terminal so interactive prompts (depotdownloader's password / Steam Guard) work exactly as they do without logging. """ logger.debug("$ %s", shlex.join(cmd)) @@ -139,7 +139,7 @@ def run_logged( except subprocess.TimeoutExpired as exc: # Record what the child managed to say before it hung, then re-raise. # Losing it here defeats the whole point of the log in precisely the - # case where it matters most -- a steamcmd hang at the 3600s timeout. + # case where it matters most -- a download hung until its timeout. output = _log_output(logger, chunks, started, returncode=None) process.wait() exc.output = output.encode() @@ -177,7 +177,7 @@ def _read_pty(fd: int, size: int) -> bytes: def _stdin_for_child(slave_fd: int) -> int: """The child's stdin: the real terminal when there is one, else the pty. - steamcmd needs a readable stdin to prompt for a password, but `sys.stdin` + depotdownloader needs a readable stdin to prompt for a password, but `sys.stdin` is not always a real file descriptor -- under pytest's capture, systemd, or a piped invocation it has no fileno(). Falling back to the pty's slave end keeps the child on a terminal-like stdin instead of crashing. diff --git a/_installer_import/steam_installer/target.py b/_installer_import/steam_installer/target.py index 6d956ba..3bf6ec2 100644 --- a/_installer_import/steam_installer/target.py +++ b/_installer_import/steam_installer/target.py @@ -8,8 +8,8 @@ Four accepted forms, all resolving to the same app: BeamNG.drive BeamNG -Whatever the input, the appid is always confirmed through steamcmd's own -`app_info_print` before anything acts on it, so the name and installdir used +Whatever the input, the appid is always confirmed against Steam's own app +cache (`appcache/appinfo.vdf`) before anything acts on it, so the name and installdir used downstream come from Steam rather than from a search index that ranks "Portal 2" above "Portal". """ diff --git a/_installer_import/steam_installer/vdf.py b/_installer_import/steam_installer/vdf.py index b73bec3..850680b 100644 --- a/_installer_import/steam_installer/vdf.py +++ b/_installer_import/steam_installer/vdf.py @@ -1,7 +1,7 @@ """Minimal reader/writer for Valve's KeyValues (VDF) text format. -Steam uses this format for appmanifest_*.acf, libraryfolders.vdf, and -steamcmd's `app_info_print` output. Only the plain-text variant is +Steam uses this format for appmanifest_*.acf and libraryfolders.vdf. +(Its binary cousin, appcache/appinfo.vdf, is read by :mod:`appinfo_cache`.) Only the plain-text variant is supported (no binary VDF) -- that covers every file this project touches. """ diff --git a/_installer_import/tests/_proton_fixture.py b/_installer_import/tests/_proton_fixture.py new file mode 100644 index 0000000..dec074c --- /dev/null +++ b/_installer_import/tests/_proton_fixture.py @@ -0,0 +1,121 @@ +"""A tiny fake CachyOS repo: a signed proton package, its db and the signing key. + +Built for real (zstd tars, a gzip .MTREE, a gpg detached signature from a +throwaway ed25519 key) so the fetcher's checks run against the same formats +the real mirror serves. +""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import subprocess +import tarfile +from dataclasses import dataclass +from pathlib import Path + +PREFIX = "usr/share/steam/compatibilitytools.d/proton-cachyos-native/" +VERSION = "1:11.0.20261015-1" +FILENAME = f"proton-cachyos-native-{VERSION}-x86_64.pkg.tar.zst" +FILES = { + "toolmanifest.vdf": b'"manifest"\n{\n "commandline" "/proton %verb%"\n}\n', + "proton": b"#!/usr/bin/env python3\n", + "files/bin/wine": b"\x7fELF wine", + "user settings.py": b"# a name with a space\n", +} +LINKS = {"files/bin/msidb": "wine"} + + +@dataclass +class Repo: + root: Path # served as the mirror: holds cachyos.db, the package, the .sig + key: Path + fingerprint: str + sha256: str + gpg_home: Path + + @property + def db_url(self) -> str: + return (self.root / "cachyos.db").as_uri() + + @property + def package(self) -> Path: + return self.root / FILENAME + + +def _mtree() -> bytes: + lines = ["#mtree", "/set type=file uid=0 gid=0 mode=644"] + lines.append("./.INSTALL time=1.0 size=3 sha256digest=00") + lines.append(f"./{PREFIX.rstrip('/')} time=1.0 mode=755 type=dir") + for name, data in FILES.items(): + digest = hashlib.sha256(data).hexdigest() + path = (PREFIX + name).replace(" ", "\\040") + lines.append(f"./{path} time=1.0 size={len(data)} sha256digest={digest}") + for name, target in LINKS.items(): + lines.append(f"./{PREFIX}{name} time=1.0 mode=777 type=link link={target}") + return gzip.compress(("\n".join(lines) + "\n").encode()) + + +def _add(tar: tarfile.TarFile, name: str, data: bytes) -> None: + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + + +def build_package(path: Path, files: dict[str, bytes] = FILES) -> None: + with tarfile.open(path, "w:zst") as tar: + _add(tar, ".INSTALL", b"rm -rf /\n") # never run, never extracted + _add(tar, ".MTREE", _mtree()) + _add(tar, "usr/bin/stray", b"outside the tool folder") + for name, data in files.items(): + _add(tar, PREFIX + name, data) + for name, target in LINKS.items(): + info = tarfile.TarInfo(PREFIX + name) + info.type = tarfile.SYMTYPE + info.linkname = target + tar.addfile(info) + + +def build_db(path: Path, version: str, sha256: str, name: str = "proton-cachyos-native") -> None: + desc = ( + f"%FILENAME%\n{name}-{version}-x86_64.pkg.tar.zst\n\n%NAME%\n{name}\n\n" + f"%VERSION%\n{version}\n\n%SHA256SUM%\n{sha256}\n\n" + ).encode() + other = b"%NAME%\nproton-cachyos-native-v3\n\n%VERSION%\n9:9-9\n\n" + with tarfile.open(path, "w:zst") as tar: + _add(tar, "proton-cachyos-native-v3-9:9-9/desc", other) + _add(tar, f"{name}-{version}/desc", desc) + + +def gpg(home: Path, *args: str) -> None: + subprocess.run( + ["gpg", "--homedir", str(home), "--batch", "--passphrase", "", *args], + check=True, + capture_output=True, + ) + + +def build_repo(tmp: Path) -> Repo: + root = tmp / "mirror" + root.mkdir() + home = tmp / "signer" + home.mkdir(mode=0o700) + gpg(home, "--quick-gen-key", "Fake CachyOS ", "ed25519", "sign", "never") + listing = subprocess.run( + ["gpg", "--homedir", str(home), "--with-colons", "--fingerprint"], + check=True, capture_output=True, text=True, + ).stdout + fingerprint = next(line.split(":")[9] for line in listing.splitlines() if line.startswith("fpr")) + key = tmp / "key.asc" + gpg(home, "--armor", "--output", str(key), "--export", fingerprint) + package = root / FILENAME + build_package(package) + gpg(home, "--output", f"{package}.sig", "--detach-sign", str(package)) + sha256 = hashlib.sha256(package.read_bytes()).hexdigest() + build_db(root / "cachyos.db", VERSION, sha256) + return Repo(root, key, fingerprint, sha256, home) + + +def stop_agent(home: Path) -> None: + subprocess.run(["gpgconf", "--homedir", str(home), "--kill", "all"], check=False) diff --git a/_installer_import/tests/conftest.py b/_installer_import/tests/conftest.py index 0956df6..79abb43 100644 --- a/_installer_import/tests/conftest.py +++ b/_installer_import/tests/conftest.py @@ -194,3 +194,34 @@ def steam(tmp_path, monkeypatch): # Prompting is tty-gated now, and pytest's stdin is not a tty. monkeypatch.setattr(cli, "_is_interactive", lambda args: True) return library, install_dir + + +@pytest.fixture(autouse=True) +def _no_live_compat_side_effects(request, tmp_path, monkeypatch): + """No test may fetch the anti-cheat list or read the real app cache. + + The modules that test those seams themselves (with their own fakes) are + exempt from the part they cover. + """ + from steam_installer import anticheat, appinfo_cache + + def offline(): + raise OSError("tests are offline") + + module = request.node.module.__name__ + if not request.node.name.startswith("test_anticheat_download"): + monkeypatch.setattr(anticheat, "_download", offline) + monkeypatch.setattr(anticheat, "CACHE", tmp_path / "no-anticheat-cache.json") + if module != "test_appinfo_cache": + monkeypatch.setattr( + appinfo_cache, "default_path", lambda: tmp_path / "no-appinfo.vdf" + ) + # The live Steam client is never seen, stopped or started by a test: an + # unstubbed --apply run once shut down and relaunched kuhy's real Steam. + from steam_installer import steamctl + + if module in ("test_steamctl", "test_recognize"): + return # these fake subprocess and /proc themselves + monkeypatch.setattr(recognize, "steam_client_pids", list) + monkeypatch.setattr(steamctl, "stop", lambda *a, **k: pytest.fail("real steam stop")) + monkeypatch.setattr(steamctl, "start", lambda *a, **k: pytest.fail("real steam start")) diff --git a/_installer_import/tests/test_appinfo.py b/_installer_import/tests/test_appinfo.py index 413114c..e509ac9 100644 --- a/_installer_import/tests/test_appinfo.py +++ b/_installer_import/tests/test_appinfo.py @@ -202,3 +202,10 @@ def test_get_build_id_handles_non_dict_sections(monkeypatch): def test_get_build_id_returns_none_when_buildid_not_a_string(monkeypatch): _app_block(monkeypatch, {"depots": {"branches": {"public": {"buildid": {}}}}}) assert appinfo.get_build_id(1) is None + + +def test_a_shared_install_without_an_owner_is_still_shared(monkeypatch): + _app_block(monkeypatch, {"depots": {"5": {"sharedinstall": "1"}, "6": {"depotfromapp": "228980"}}}) + plan = appinfo.get_depot_plan(1) + assert plan.shared == {"5", "6"} + assert plan.shared_owner == {"6": "228980"} diff --git a/_installer_import/tests/test_appinfo_cache.py b/_installer_import/tests/test_appinfo_cache.py index 18d6dc5..7e80fc1 100644 --- a/_installer_import/tests/test_appinfo_cache.py +++ b/_installer_import/tests/test_appinfo_cache.py @@ -1,10 +1,10 @@ """Steam's local binary app-info cache, read without steamcmd.""" import pytest -from steam_installer import appinfo_cache - from _appinfo_blob import encode +from steam_installer import appinfo_cache + def _write(tmp_path, apps): path = tmp_path / "appinfo.vdf" diff --git a/_installer_import/tests/test_cli_finish.py b/_installer_import/tests/test_cli_finish.py index c320d43..2942006 100644 --- a/_installer_import/tests/test_cli_finish.py +++ b/_installer_import/tests/test_cli_finish.py @@ -4,6 +4,8 @@ import subprocess import sys import pytest +from _cli_helpers import ARGV, _stub_download + from steam_installer import ( appinfo, cli, @@ -11,8 +13,6 @@ from steam_installer import ( vdf, ) -from _cli_helpers import ARGV, _stub_download - pytestmark = pytest.mark.usefixtures("cli_logs") diff --git a/_installer_import/tests/test_compat_decide.py b/_installer_import/tests/test_compat_decide.py new file mode 100644 index 0000000..1639dc9 --- /dev/null +++ b/_installer_import/tests/test_compat_decide.py @@ -0,0 +1,164 @@ +"""Which games get the native Proton, and why the others keep Valve's default.""" + +import json + +import pytest + +from steam_installer import anticheat, compat_decide + +WINDOWS_ONLY = { + "common": {"oslist": "windows"}, + "depots": {"11": {"config": {"oslist": "windows"}}, "12": {}}, + "config": {"launch": {"0": {"config": {"oslist": "windows"}}}}, +} + + +def _decide(block=WINDOWS_ONLY, **kw): + args = {"anticheat": set(), "overrides": {}, "owned_dlc": False, "tool_ready": True} + args.update(kw) + return compat_decide.decide(10, block, **args) + + +def test_a_plain_windows_game_is_mapped(): + decision = _decide() + assert decision.map + assert decision.describe() == "map" + + +@pytest.mark.parametrize( + ("block", "reason"), + [ + ({**WINDOWS_ONLY, "common": {"oslist": "windows,linux"}}, "native"), + ({**WINDOWS_ONLY, "depots": {"11": {"config": {"oslist": "linux"}}}}, "native"), + ( + {**WINDOWS_ONLY, "config": {"launch": {"0": {"config": {"oslist": "linux"}}}}}, + "native", + ), + ({**WINDOWS_ONLY, "common": {"oslist": "windows,macos"}}, "not Windows-only"), + ({**WINDOWS_ONLY, "common": {}}, "no OS list"), + ({"common": "x"}, "no OS list"), + ({**WINDOWS_ONLY, "depots": {"11": {"depotfromapp": "228980"}}}, "shared depots"), + ({**WINDOWS_ONLY, "depots": {"11": {"sharedinstall": "1"}}}, "shared depots"), + ], +) +def test_unsafe_games_keep_valves_default(block, reason): + decision = _decide(block) + assert not decision.map + assert reason in decision.reasons + + +def test_odd_shapes_are_ignored_not_crashed_on(): + block = { + "common": {"oslist": "windows"}, + "depots": {"branches": {"public": {}}, "11": "scalar", "12": {"config": "x"}}, + "config": {"launch": "scalar"}, + } + assert _decide(block).map + assert _decide({**block, "config": {"launch": {"0": "x", "1": {"config": "y"}}}}).map + assert _decide({**block, "depots": "scalar", "config": "scalar"}).map + + +def test_anti_cheat_whatever_its_status(): + assert _decide(anticheat={10}).reasons == ["anti-cheat"] + + +def test_no_anti_cheat_list_means_no_mapping(): + assert _decide(anticheat=None).reasons == ["anti-cheat list unavailable"] + + +def test_owned_dlc_is_left_on_valves_default_until_tested_live(): + assert _decide(owned_dlc=True).reasons == ["owned DLC"] + + +def test_override_and_missing_tool(): + assert _decide(overrides={"10": "crashes at intro"}).reasons == [ + "override: crashes at intro" + ] + assert _decide(tool_ready=False).reasons == ["native Proton not installed"] + + +def test_every_reason_is_reported(): + block = {**WINDOWS_ONLY, "common": {"oslist": "windows,linux"}} + decision = _decide(block, anticheat={10}) + assert decision.describe() == "Valve default: anti-cheat, native" + + +def test_overrides_file_is_tracked_and_valid(): + assert compat_decide.load_overrides() == {} + + +def test_overrides_file_must_map_ids_to_reasons(tmp_path): + path = tmp_path / "o.json" + path.write_text(json.dumps({"10": "why"})) + assert compat_decide.load_overrides(path) == {"10": "why"} + for bad in ([1], {"x": "why"}, {"10": 3}): + path.write_text(json.dumps(bad)) + with pytest.raises(ValueError, match="compat_overrides"): + compat_decide.load_overrides(path) + + +def test_tool_ready_needs_the_manifest_and_the_pinned_stamp(tmp_path): + tool = tmp_path / "proton-cachyos-native" + assert not compat_decide.tool_ready(tool, "1:2-3") + tool.mkdir() + (tool / "toolmanifest.vdf").write_text("x") + assert not compat_decide.tool_ready(tool, "1:2-3") + (tool / "VERSION").write_text("1:2-2\n") + assert not compat_decide.tool_ready(tool, "1:2-3") + (tool / "VERSION").write_text("1:2-3\n") + assert compat_decide.tool_ready(tool, "1:2-3") + + +GAMES = [ + {"name": "CS", "storeIds": {"steam": "730"}, "status": "Supported"}, + {"name": "Epic only", "storeIds": {"epic": "x"}}, + {"name": "Broken", "storeIds": "x"}, + "not a game", +] + + +def test_anticheat_list_is_fetched_and_cached(tmp_path, monkeypatch): + cache = tmp_path / "ac.json" + monkeypatch.setattr(anticheat, "_download", lambda: json.dumps(GAMES).encode()) + assert anticheat.steam_ids(cache) == {730} + assert json.loads(cache.read_text()) == GAMES + + +def test_anticheat_falls_back_to_the_last_good_copy(tmp_path, monkeypatch): + cache = tmp_path / "ac.json" + cache.write_text(json.dumps(GAMES)) + + def offline(): + raise OSError("no network") + + monkeypatch.setattr(anticheat, "_download", offline) + assert anticheat.steam_ids(cache) == {730} + + +def test_anticheat_with_no_fetch_and_no_cache_is_unknown(tmp_path, monkeypatch): + monkeypatch.setattr(anticheat, "_download", lambda: b"{not json") + assert anticheat.steam_ids(tmp_path / "ac.json") is None + (tmp_path / "ac.json").write_text('{"a": 1}') + assert anticheat.steam_ids(tmp_path / "ac.json") is None + + +def test_anticheat_download_reads_the_upstream_url(monkeypatch): + seen = {} + + class _Response: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return b"[]" + + def fake_urlopen(url, timeout): + seen["url"] = url + return _Response() + + monkeypatch.setattr(anticheat.urllib.request, "urlopen", fake_urlopen) + assert anticheat._download() == b"[]" + assert "AreWeAntiCheatYet" in seen["url"] diff --git a/_installer_import/tests/test_compat_map.py b/_installer_import/tests/test_compat_map.py new file mode 100644 index 0000000..27638cd --- /dev/null +++ b/_installer_import/tests/test_compat_map.py @@ -0,0 +1,198 @@ +"""Per-game CompatToolMapping entries, inserted into config.vdf as text.""" + +import pytest + +from steam_installer import compat_map + +HEAD = '''"InstallConfigStore" +{ +\t"Software" +\t{ +\t\t"Valve" +\t\t{ +\t\t\t"Steam" +\t\t\t{ +\t\t\t\t"SecretToken"\t\t"do-not-print-me" +''' +TAIL = '''\t\t\t} +\t\t} +\t} +\t"Music" +\t{ +\t\t"CompatToolMapping"\t\t"decoy outside the Steam block" +\t} +} +''' + + +def _entry(appid, name, priority, indent="\t\t\t\t\t"): + return ( + f'{indent}"{appid}"\n{indent}{{\n' + f'{indent}\t"name"\t\t"{name}"\n{indent}\t"config"\t\t""\n' + f'{indent}\t"priority"\t\t"{priority}"\n{indent}}}\n' + ) + + +def _config(tmp_path, mapping=None): + body = HEAD + if mapping is not None: + body += '\t\t\t\t"CompatToolMapping"\n\t\t\t\t{\n' + mapping + "\t\t\t\t}\n" + path = tmp_path / "config.vdf" + path.write_text(body + TAIL) + path.chmod(0o600) + return path + + +@pytest.fixture(autouse=True) +def _steam_closed(monkeypatch): + monkeypatch.setattr(compat_map.recognize, "steam_client_pids", list) + + +def test_adds_the_block_when_there_is_none(tmp_path): + path = _config(tmp_path) + before = path.read_text() + + outcome = compat_map.map_app(path, 1201270) + + assert outcome.action == "mapped" + text = path.read_text() + assert compat_map.current(path, 1201270) == ("proton-cachyos-native", "250") + # Everything Steam wrote is still there byte for byte. + assert text.replace(_mapping_block(1201270), "") == before + assert path.stat().st_mode & 0o777 == 0o600 + + +def _mapping_block(appid): + return ( + '\t\t\t\t"CompatToolMapping"\n\t\t\t\t{\n' + + _entry(appid, "proton-cachyos-native", "250") + + "\t\t\t\t}\n" + ) + + +def test_adds_an_entry_to_an_existing_block(tmp_path): + path = _config(tmp_path, _entry(10, "proton_9", "250")) + assert compat_map.map_app(path, 20).action == "mapped" + assert compat_map.current(path, 10) == ("proton_9", "250") + assert compat_map.current(path, 20) == ("proton-cachyos-native", "250") + + +def test_replaces_valves_own_choice(tmp_path): + path = _config(tmp_path, _entry(20, "proton_experimental", "100")) + outcome = compat_map.map_app(path, 20) + assert outcome.action == "mapped" + assert "proton_experimental" in outcome.detail + assert compat_map.current(path, 20) == ("proton-cachyos-native", "250") + assert path.read_text().count('"20"') == 1 + + +def test_replaces_an_empty_entry(tmp_path): + path = _config(tmp_path, _entry(20, "", "250")) + assert compat_map.map_app(path, 20).action == "mapped" + assert compat_map.current(path, 20) == ("proton-cachyos-native", "250") + + +def test_keeps_a_tool_kuhy_picked_himself(tmp_path): + path = _config(tmp_path, _entry(20, "GE-Proton10", "250")) + before = path.read_text() + outcome = compat_map.map_app(path, 20) + assert outcome.action == "kept" + assert "GE-Proton10" in outcome.detail + assert path.read_text() == before + + +def test_a_second_run_changes_nothing(tmp_path): + path = _config(tmp_path) + compat_map.map_app(path, 20) + once = path.read_text() + assert compat_map.map_app(path, 20).action == "already" + assert path.read_text() == once + + +def test_refuses_while_steam_runs(tmp_path, monkeypatch): + path = _config(tmp_path) + before = path.read_text() + monkeypatch.setattr(compat_map.recognize, "steam_client_pids", lambda: [7]) + with pytest.raises(compat_map.CompatMapError, match="Steam is running"): + compat_map.map_app(path, 20) + assert path.read_text() == before + + +@pytest.mark.parametrize( + "text", + [ + '"InstallConfigStore"\n{\n}\n', + '"InstallConfigStore"\n{\n\t"Software" { }\n}\n', + '"InstallConfigStore"\n{\n\t}\n\t}\n', + '{\n}\n', + ], +) +def test_an_unexpected_layout_is_refused_without_echoing_lines(tmp_path, text): + path = tmp_path / "config.vdf" + path.write_text(text + '"x"\t"do-not-print-me"\n') + with pytest.raises(compat_map.CompatMapError) as excinfo: + compat_map.map_app(path, 20) + assert "do-not-print-me" not in str(excinfo.value) + + +def test_missing_config_is_an_error(tmp_path): + with pytest.raises(compat_map.CompatMapError, match="cannot read"): + compat_map.current(tmp_path / "nope.vdf", 1) + + +def test_current_is_none_for_an_unmapped_app(tmp_path): + assert compat_map.current(_config(tmp_path), 20) is None + assert compat_map.current(_config(tmp_path, _entry(10, "x", "1")), 20) is None + + +def test_keys_match_case_insensitively(tmp_path): + path = _config(tmp_path) + path.write_text(path.read_text().replace('"Steam"', '"steam"')) + assert compat_map.map_app(path, 20).action == "mapped" + assert compat_map.current(path, 20) == ("proton-cachyos-native", "250") + + +def test_config_path_is_under_the_steam_root(tmp_path): + assert compat_map.config_path(tmp_path) == tmp_path / "config" / "config.vdf" + + +def test_a_value_spanning_lines_is_read_as_one_value(tmp_path): + """Steam stores some values with embedded newlines (seen in the real file).""" + path = _config(tmp_path) + path.write_text(path.read_text().replace( + '"SecretToken"\t\t"do-not-print-me"', '"SecretToken"\t\t"line one\nline {two}"')) + assert compat_map.map_app(path, 20).action == "mapped" + assert compat_map.current(path, 20) == ("proton-cachyos-native", "250") + + +def test_an_entry_sharing_a_line_with_its_key_is_refused(tmp_path): + path = _config(tmp_path, '\t\t\t\t\t"20" {\n\t\t\t\t\t\t"name" "x"\n\t\t\t\t\t}\n') + with pytest.raises(compat_map.CompatMapError, match="unexpected layout"): + compat_map.map_app(path, 20) + + +def test_an_entry_closing_on_a_shared_line_is_refused(tmp_path): + path = _config(tmp_path, '\t\t\t\t\t"20"\n\t\t\t\t\t{\n\t\t\t\t\t\t"name" "" }\n') + with pytest.raises(compat_map.CompatMapError, match="unexpected layout"): + compat_map.map_app(path, 20) + + +def test_unquoted_tokens_are_refused(tmp_path): + path = _config(tmp_path) + path.write_text(path.read_text().replace('"SecretToken"', "SecretToken")) + with pytest.raises(compat_map.CompatMapError, match="line 9: unexpected layout"): + compat_map.current(path, 20) + + +def test_a_failed_replace_leaves_the_file_and_no_temp(tmp_path, monkeypatch): + path = _config(tmp_path) + before = path.read_text() + + def fail(src, dst): + raise OSError("disk full") + + monkeypatch.setattr(compat_map.os, "replace", fail) + with pytest.raises(OSError, match="disk full"): + compat_map.map_app(path, 20) + assert path.read_text() == before + assert sorted(p.name for p in tmp_path.iterdir()) == ["config.vdf"] diff --git a/_installer_import/tests/test_install_sh.py b/_installer_import/tests/test_install_sh.py new file mode 100644 index 0000000..d431b1e --- /dev/null +++ b/_installer_import/tests/test_install_sh.py @@ -0,0 +1,45 @@ +"""install.sh --check-only: what a run needs, with steamcmd no longer on the list.""" + +import os +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def _stub(bin_dir: Path, name: str, script: str) -> None: + path = bin_dir / name + path.write_text(f"#!/bin/bash\n{script}\n") + path.chmod(0o755) + + +def _check_only(tmp_path, libibus_installed: bool): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _stub(bin_dir, "depotdownloader", "exit 0") + _stub(bin_dir, "pacman", f'[[ "$*" == "-Q libibus" ]] && exit {0 if libibus_installed else 1}; exit 0') + env = {**os.environ, "PATH": f"{bin_dir}:/usr/bin:/bin", "HOME": str(tmp_path)} + return subprocess.run( + ["bash", str(ROOT / "install.sh"), "--check-only"], + capture_output=True, text=True, env=env, timeout=30, check=False, + ) + + +def test_missing_libibus_is_reported_and_steamcmd_is_not_needed(tmp_path): + result = _check_only(tmp_path, libibus_installed=False) + assert result.returncode == 1 + assert "missing: libibus" in result.stderr + assert "steamcmd" not in result.stdout + result.stderr + + +def test_all_present_without_steamcmd(tmp_path): + result = _check_only(tmp_path, libibus_installed=True) + assert result.returncode == 0, result.stderr + assert "All dependencies present." in result.stdout + + +def test_run_sh_rechecks_when_the_proton_pin_changes(): + """A stale stamp must not hide a lock bump or a deleted tool folder.""" + text = (ROOT / "run.sh").read_text() + assert "proton-native.lock" in text + assert "steam_installer.proton_native --check" in text diff --git a/_installer_import/tests/test_map_installed.py b/_installer_import/tests/test_map_installed.py new file mode 100644 index 0000000..6449a64 --- /dev/null +++ b/_installer_import/tests/test_map_installed.py @@ -0,0 +1,197 @@ +"""Mapping games to the native Proton: after an install, and for the whole library.""" + +import pytest +from _appinfo_blob import encode +from _cli_helpers import ARGV, _stub_download + +from steam_installer import ( + anticheat, + cli, + compat_apply, + compat_map, + proton_pkg, + recognize, + steamctl, + vdf, +) + +pytestmark = pytest.mark.usefixtures("cli_logs") + +LOCKED = proton_pkg.read_lock().version +CONFIG = '"InstallConfigStore"\n{\n\t"Software"\n\t{\n\t\t"Valve"\n\t\t{\n\t\t\t"Steam"\n\t\t\t{\n\t\t\t}\n\t\t}\n\t}\n}\n' +PLAIN = {"common": {"name": "Plain", "oslist": "windows", "type": "Game"}, + "depots": {"11": {"config": {"oslist": "windows"}}}} +NATIVE = {"common": {"name": "Native", "oslist": "windows,linux", "type": "Game"}} +SHARED = {"common": {"name": "Shared", "oslist": "windows", "type": "Game"}, + "depots": {"31": {"depotfromapp": "228980"}}} +CS = {"common": {"name": "CS", "oslist": "windows,linux", "type": "Game"}} +TOOL = {"common": {"name": "Proton Experimental", "type": "Tool"}} + + +@pytest.fixture +def home(tmp_path, monkeypatch): + """A Steam root with config.vdf, the stamped tool, an app cache and anti-cheat list.""" + root = tmp_path / "Steam" + (root / "config").mkdir(parents=True) + (root / "config" / "config.vdf").write_text(CONFIG) + tool = root / "compatibilitytools.d" / "proton-cachyos-native" + tool.mkdir(parents=True) + (tool / "toolmanifest.vdf").write_text("x") + (tool / "VERSION").write_text(LOCKED + "\n") + cache = tmp_path / "appinfo.vdf" + cache.write_bytes(encode({10: PLAIN, 20: NATIVE, 30: SHARED, 730: CS, 1493710: TOOL})) + monkeypatch.setattr(compat_apply.appinfo.appinfo_cache, "default_path", lambda: cache) + monkeypatch.setattr(anticheat, "_download", lambda: b'[{"storeIds": {"steam": "730"}}]') + steamapps = root / "steamapps" + steamapps.mkdir() + for appid in (10, 20, 1493710): + (steamapps / f"appmanifest_{appid}.acf").write_text( + vdf.dumps("AppState", {"appid": str(appid), "InstalledDepots": {}})) + monkeypatch.setattr(cli, "find_steam_root", lambda: root) + return root + + +def _mapped(root, appid): + return compat_map.current(root / "config" / "config.vdf", appid) + + +def test_dry_run_lists_decisions_and_writes_nothing(home, capsys): + before = (home / "config" / "config.vdf").read_bytes() + assert cli.main(["--map-installed"]) == 0 + out = capsys.readouterr().out + assert "10 Plain: map" in out + assert "20 Native: Valve default: native" in out + assert "1493710" not in out # a tool, not a game + assert "Dry run" in out + assert (home / "config" / "config.vdf").read_bytes() == before + + +def test_appids_decides_any_cached_app(home, capsys): + assert cli.main(["--map-installed", "--appids", "730,20,30,10,999"]) == 0 + out = capsys.readouterr().out + assert "730 CS: Valve default: anti-cheat, native" in out + assert "30 Shared: Valve default: shared depots" in out + assert "10 Plain: map" in out + assert "999: not in Steam's app cache" in out + + +def test_apply_maps_the_safe_ones_and_a_rerun_changes_nothing(home, capsys): + assert cli.main(["--map-installed", "--apply", "--yes"]) == 0 + assert _mapped(home, 10) == ("proton-cachyos-native", "250") + assert _mapped(home, 20) is None + once = (home / "config" / "config.vdf").read_bytes() + capsys.readouterr() + assert cli.main(["--map-installed", "--apply", "--yes"]) == 0 + assert "10 Plain: map -- already mapped" in capsys.readouterr().out + assert (home / "config" / "config.vdf").read_bytes() == once + + +def test_apply_closes_steam_first_and_restarts_it(home, monkeypatch): + events = [] + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [] if events else [5]) + monkeypatch.setattr(steamctl, "stop", lambda logger: events.append("stop") or True) + monkeypatch.setattr(steamctl, "start", lambda logger: events.append("start")) + assert cli.main(["--map-installed", "--apply", "--yes"]) == 0 + assert events == ["stop", "start"] + assert _mapped(home, 10) is not None + + +def test_apply_declined_writes_nothing(home, monkeypatch): + monkeypatch.setattr(recognize, "steam_client_pids", lambda: [5]) + monkeypatch.setattr("builtins.input", lambda *a: "n") + assert cli.main(["--map-installed", "--apply"]) == 1 + assert _mapped(home, 10) is None + + +def test_a_kept_choice_and_a_write_error_are_reported(home, monkeypatch, capsys): + config = home / "config" / "config.vdf" + compat_map.map_app(config, 10) + config.write_text(config.read_text().replace("proton-cachyos-native", "GE-Proton10")) + assert cli.main(["--map-installed", "--apply", "--yes"]) == 0 + assert "kept: already mapped to GE-Proton10" in capsys.readouterr().out + config.write_text("garbage {") + assert cli.main(["--map-installed", "--apply", "--yes"]) == 1 + + +def test_owned_dlc_in_the_manifest_keeps_valves_default(home, capsys): + manifest = home / "steamapps" / "appmanifest_10.acf" + manifest.write_text(vdf.dumps("AppState", {"appid": "10", "InstalledDepots": { + "11": {"manifest": "1"}, "12": {"manifest": "2", "dlcappid": "13"}}})) + assert cli.main(["--map-installed"]) == 0 + assert "10 Plain: Valve default: owned DLC" in capsys.readouterr().out + + +def test_valve_proton_only_goes_with_an_install(home): + with pytest.raises(SystemExit): + cli.main(["--map-installed", "--valve-proton"]) + + +def test_an_install_maps_the_game_while_steam_is_closed(steam, monkeypatch, capsys, tmp_path): + library, install_dir = steam + _with_compat_home(library.path, monkeypatch, tmp_path) + _stub_download(monkeypatch) + monkeypatch.setattr(compat_apply.appinfo, "_fetch_app_block", lambda appid: PLAIN) + _proven(monkeypatch, install_dir) + assert cli.main(ARGV) == 0 + assert _mapped(library.path, 1771300) == ("proton-cachyos-native", "250") + assert "mapped to proton-cachyos-native" in capsys.readouterr().out + + +def test_valve_proton_keeps_valves_default_for_one_run(steam, monkeypatch, capsys, tmp_path): + library, install_dir = steam + _with_compat_home(library.path, monkeypatch, tmp_path) + _stub_download(monkeypatch) + monkeypatch.setattr(compat_apply.appinfo, "_fetch_app_block", lambda appid: PLAIN) + _proven(monkeypatch, install_dir) + assert cli.main([*ARGV, "--valve-proton"]) == 0 + assert _mapped(library.path, 1771300) is None + assert "Valve default: override: --valve-proton" in capsys.readouterr().out + + +def test_a_mapping_failure_leaves_the_install_on_valves_default(steam, monkeypatch, capsys, tmp_path): + """The install itself succeeded; the game still runs on Valve's Proton.""" + library, install_dir = steam + _with_compat_home(library.path, monkeypatch, tmp_path) + (library.path / "config" / "config.vdf").write_text("broken {") + _stub_download(monkeypatch) + monkeypatch.setattr(compat_apply.appinfo, "_fetch_app_block", lambda appid: PLAIN) + _proven(monkeypatch, install_dir) + assert cli.main(ARGV) == 0 + assert "stays on Valve's default" in capsys.readouterr().out + + +def test_an_owned_dlc_download_keeps_valves_default(steam, monkeypatch, capsys, tmp_path): + library, install_dir = steam + _with_compat_home(library.path, monkeypatch, tmp_path) + _stub_download(monkeypatch) + monkeypatch.setattr(compat_apply.appinfo, "_fetch_app_block", lambda appid: PLAIN) + _proven(monkeypatch, install_dir, dlc={"1771303": "9"}) + assert cli.main(ARGV) == 0 + assert "Valve default: owned DLC" in capsys.readouterr().out + + +def _with_compat_home(root, monkeypatch, tmp_path): + (root / "config").mkdir(parents=True, exist_ok=True) + (root / "config" / "config.vdf").write_text(CONFIG) + tool = root / "compatibilitytools.d" / "proton-cachyos-native" + tool.mkdir(parents=True) + (tool / "toolmanifest.vdf").write_text("x") + (tool / "VERSION").write_text(LOCKED + "\n") + monkeypatch.setattr(anticheat, "_download", lambda: b"[]") + + +def _proven(monkeypatch, install_dir, dlc=None): + (gid,) = recognize.discover_cached_depot_manifests(install_dir).values() + plan = compat_apply.appinfo.DepotPlan({"1771303"}, set(), manifests={"1771303": gid}, + dlc=dict(dlc or {})) + monkeypatch.setattr(compat_apply.appinfo, "get_depot_plan", lambda appid, language: plan) + + +def test_a_failed_write_stops_the_run(home, monkeypatch, capsys): + def boom(config, appid): + raise compat_map.CompatMapError("config.vdf line 3: unexpected layout") + + monkeypatch.setattr(compat_map, "map_app", boom) + (home / "steamapps" / "appmanifest_notanid.acf").write_text("x") + assert cli.main(["--map-installed", "--apply", "--yes"]) == 1 + assert "Mapping stopped: config.vdf line 3" in capsys.readouterr().out diff --git a/_installer_import/tests/test_proton_native.py b/_installer_import/tests/test_proton_native.py new file mode 100644 index 0000000..6272089 --- /dev/null +++ b/_installer_import/tests/test_proton_native.py @@ -0,0 +1,211 @@ +"""Fetch, verify and install the native Proton, never running its .INSTALL.""" + +import os +import shutil + +import pytest +from _proton_fixture import ( + FILES, + VERSION, + build_db, + build_package, + build_repo, + gpg, + stop_agent, +) + +from steam_installer import proton_native, proton_pkg + + +@pytest.fixture(scope="module") +def repo(tmp_path_factory): + built = build_repo(tmp_path_factory.mktemp("repo")) + yield built + stop_agent(built.gpg_home) + + +@pytest.fixture +def mirror(repo, tmp_path): + """A private copy of the fake mirror a test may tamper with.""" + root = tmp_path / "mirror" + shutil.copytree(repo.root, root) + return root + + +@pytest.fixture +def setup(repo, mirror, tmp_path, monkeypatch): + tools = tmp_path / "compatibilitytools.d" + tools.mkdir() + lock = proton_pkg.Lock( + "proton-cachyos-native", (mirror / "cachyos.db").as_uri(), VERSION, repo.sha256 + ) + monkeypatch.setattr(proton_native, "in_use", lambda tool: []) + + def run(**kw): + return proton_native.ensure( + tools, lock, key=repo.key, fingerprint=repo.fingerprint, **kw + ) + + return tools, lock, run + + +def _tree(tool): + return sorted(str(p.relative_to(tool)) for p in tool.rglob("*")) + + +def test_a_fresh_install_extracts_only_the_tool_and_stamps_it(setup): + tools, _, run = setup + status = run() + tool = tools / "proton-cachyos-native" + assert status.startswith("installed") + assert "signature" in status and "sha256" in status + assert (tool / "VERSION").read_text() == VERSION + "\n" + assert (tool / "files/bin/wine").read_bytes() == FILES["files/bin/wine"] + assert os.readlink(tool / "files/bin/msidb") == "wine" + assert not (tools / ".INSTALL").exists() + assert not any(".INSTALL" in p or "stray" in p for p in _tree(tools)) + assert sorted(os.listdir(tools)) == ["proton-cachyos-native"] + + +def test_a_current_stamp_needs_no_network(setup, monkeypatch): + _, _, run = setup + run() + monkeypatch.setattr(proton_native, "_download", lambda url, dest: pytest.fail()) + assert run() == f"current: {VERSION}" + + +def _assert_fails_untouched(run, tools, match): + old = tools / "proton-cachyos-native" + old.mkdir(exist_ok=True) + (old / "keep").write_text("old build") + with pytest.raises(proton_pkg.PkgError, match=match): + run() + assert (old / "keep").read_text() == "old build" + assert sorted(os.listdir(tools)) == ["proton-cachyos-native"] + + +def test_a_tampered_signature_fails_closed(setup, mirror): + tools, _, run = setup + sig = next(mirror.glob("*.sig")) + data = bytearray(sig.read_bytes()) + data[-5] ^= 0xFF + sig.write_bytes(bytes(data)) + _assert_fails_untouched(run, tools, "signature") + + +def test_a_signature_from_another_key_fails_closed(setup, mirror, tmp_path): + tools, _, run = setup + other = tmp_path / "other" + other.mkdir(mode=0o700) + gpg(other, "--quick-gen-key", "Mallory ", "ed25519", "sign", "never") + package = next(mirror.glob("*.pkg.tar.zst")) + (mirror / f"{package.name}.sig").unlink() + gpg(other, "--output", f"{package}.sig", "--detach-sign", str(package)) + stop_agent(other) + _assert_fails_untouched(run, tools, "signature") + + +def test_a_package_that_differs_from_the_pinned_sha256_fails_closed(setup, mirror, repo): + tools, _, run = setup + package = next(mirror.glob("*.pkg.tar.zst")) + build_package(package, {**FILES, "proton": b"evil"}) + gpg(repo.gpg_home, "--yes", "--output", f"{package}.sig", "--detach-sign", str(package)) + _assert_fails_untouched(run, tools, "sha256") + + +def test_a_db_sha256_that_disagrees_with_the_lock_fails_closed(setup, mirror): + tools, _, run = setup + build_db(mirror / "cachyos.db", VERSION, "0" * 64) + _assert_fails_untouched(run, tools, "sha256") + + +def test_a_newer_package_in_the_repo_asks_for_a_lock_bump(setup, mirror, repo): + tools, _, run = setup + build_db(mirror / "cachyos.db", "1:11.0.20261101-1", repo.sha256) + _assert_fails_untouched(run, tools, "bump") + + +def test_a_repo_without_the_package_fails(setup, mirror, repo): + tools, _, run = setup + build_db(mirror / "cachyos.db", VERSION, repo.sha256, name="something-else") + _assert_fails_untouched(run, tools, "no proton-cachyos-native") + + +def _existing_copy(tools): + tool = tools / "proton-cachyos-native" + for name, data in FILES.items(): + (tool / name).parent.mkdir(parents=True, exist_ok=True) + (tool / name).write_bytes(data) + os.symlink("wine", tool / "files/bin/msidb") + (tool / "__pycache__").mkdir() + (tool / "__pycache__" / "x.pyc").write_bytes(b"made by running it") + return tool + + +def test_an_identical_copy_is_adopted_without_extracting(setup, monkeypatch): + tools, _, run = setup + tool = _existing_copy(tools) + monkeypatch.setattr(proton_pkg, "extract", lambda *a: pytest.fail("extracted")) + status = run() + assert status.startswith(f"adopted: all {len(FILES) + 1} files match .MTREE (signature") + assert (tool / "VERSION").read_text() == VERSION + "\n" + + +@pytest.mark.parametrize( + ("change", "where"), + [ + (lambda t: (t / "files/bin/wine").write_bytes(b"patched"), "files/bin/wine"), + (lambda t: (t / "user_settings.py").write_text("x"), "user_settings.py"), + (lambda t: (t / "proton").unlink(), "proton"), + (lambda t: (t / "files/bin/msidb").unlink() or os.symlink("x", t / "files/bin/msidb"), "msidb"), + (lambda t: (t / "files/bin/msidb").unlink(), "msidb"), + ], +) +def test_a_differing_copy_is_replaced(setup, change, where): + tools, _, run = setup + tool = _existing_copy(tools) + change(tool) + status = run() + assert status.startswith("replaced: mismatch in") and where in status + assert (tool / "files/bin/wine").read_bytes() == FILES["files/bin/wine"] + assert not (tool / "user_settings.py").exists() + assert sorted(os.listdir(tools)) == ["proton-cachyos-native"] + + +def test_replacing_is_deferred_while_steam_or_a_game_runs(setup, monkeypatch): + tools, _, run = setup + tool = _existing_copy(tools) + (tool / "proton").write_text("old") + monkeypatch.setattr(proton_native, "in_use", lambda t: [4242]) + status = run() + assert status.startswith("deferred") + assert (tool / "proton").read_text() == "old" + assert not (tool / "VERSION").exists() + + +def test_an_older_stamp_is_updated(setup): + tools, _, run = setup + tool = _existing_copy(tools) + (tool / "VERSION").write_text("1:10.0-1\n") + assert run().startswith("updated: 1:10.0-1 ->") + assert (tool / "VERSION").read_text() == VERSION + "\n" + + +def test_in_use_sees_steam_and_processes_running_from_the_tool(tmp_path, monkeypatch): + proc = tmp_path / "proc" + tool = tmp_path / "tool" + (tool / "files/bin").mkdir(parents=True) + (tool / "files/bin/wine").write_text("") + (tmp_path / "other").write_text("") + for pid, target in ( + ("10", tool / "files/bin/wine"), + ("11", tmp_path / "other"), + ("13", tmp_path / "gone"), + ): + (proc / pid).mkdir(parents=True) + os.symlink(target, proc / pid / "exe") + (proc / "self").mkdir() + (proc / "12").mkdir() + monkeypatch.setattr(proton_native.recognize, "PROC_DIR", proc) + monkeypatch.setattr(proton_native.recognize, "steam_client_pids", lambda: [99]) + assert proton_native.in_use(tool) == [99, 10] diff --git a/_installer_import/tests/test_proton_native_cli.py b/_installer_import/tests/test_proton_native_cli.py new file mode 100644 index 0000000..eae936d --- /dev/null +++ b/_installer_import/tests/test_proton_native_cli.py @@ -0,0 +1,58 @@ +"""The proton_native entry point: what install.sh and run.sh call.""" + +import subprocess +import sys + +from steam_installer import proton_native, proton_pkg + + +def test_main_reports_and_exits(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(proton_native, "ensure", lambda *a, **kw: "current: x") + monkeypatch.setattr(proton_native, "find_steam_root", lambda: tmp_path) + assert proton_native.main([]) == 0 + assert "current: x" in capsys.readouterr().out + + def boom(*a, **kw): + raise proton_pkg.PkgError("bad signature") + + monkeypatch.setattr(proton_native, "ensure", boom) + assert proton_native.main([]) == 1 + assert "bad signature" in capsys.readouterr().err + + +def test_check_reports_whether_the_stamp_matches_the_lock(tmp_path, monkeypatch): + monkeypatch.setattr(proton_native, "find_steam_root", lambda: tmp_path) + assert proton_native.main(["--check"]) == 1 + tool = tmp_path / "compatibilitytools.d" / "proton-cachyos-native" + tool.mkdir(parents=True) + (tool / "VERSION").write_text(proton_pkg.read_lock().version + "\n") + assert proton_native.main(["--check"]) == 0 + + +def test_no_steam_installation_means_nothing_to_install(monkeypatch, capsys): + def absent(): + raise FileNotFoundError("No Steam installation found") + + monkeypatch.setattr(proton_native, "find_steam_root", absent) + assert proton_native.main([]) == 0 + assert proton_native.main(["--check"]) == 0 + assert "skipped" in capsys.readouterr().out + + +def test_the_repo_can_be_pointed_at_a_test_mirror(monkeypatch, tmp_path, capsys): + seen = {} + monkeypatch.setattr(proton_native, "find_steam_root", lambda: tmp_path) + monkeypatch.setenv("SGI_PROTON_REPO", "file:///mirror/cachyos.db") + monkeypatch.setattr(proton_native, "ensure", lambda d, lock: seen.setdefault("repo", lock.repo)) + assert proton_native.main([]) == 0 + assert seen["repo"] == "file:///mirror/cachyos.db" + + +def test_module_runs_as_a_script(tmp_path, coverage_subprocess_env): + env = {**coverage_subprocess_env, "HOME": str(tmp_path)} + result = subprocess.run( + [sys.executable, "-m", "steam_installer.proton_native", "--check"], + capture_output=True, text=True, env=env, timeout=30, check=False, + ) + assert result.returncode == 0 + assert "skipped" in result.stdout diff --git a/_installer_import/tests/test_proton_pkg.py b/_installer_import/tests/test_proton_pkg.py new file mode 100644 index 0000000..68fb12c --- /dev/null +++ b/_installer_import/tests/test_proton_pkg.py @@ -0,0 +1,98 @@ +"""The package formats and checks behind the native Proton fetcher.""" + +import hashlib +import tarfile + +import pytest +from _proton_fixture import PREFIX, build_repo, stop_agent + +from steam_installer import proton_pkg + + +@pytest.fixture(scope="module") +def repo(tmp_path_factory): + built = build_repo(tmp_path_factory.mktemp("repo")) + yield built + stop_agent(built.gpg_home) + + +def test_the_real_lock_pins_the_proven_build(): + lock = proton_pkg.read_lock() + assert lock.name == "proton-cachyos-native" + assert lock.version == "1:11.0.20261015-1" + assert lock.sha256 == "25bf302f90e8e01118555b997b873c3cd06a33b5647e79d049a59e29794b43e0" + assert lock.repo == "https://mirror.cachyos.org/repo/x86_64/cachyos/cachyos.db" + + +def test_a_lock_missing_a_key_is_refused(tmp_path): + path = tmp_path / "x.lock" + path.write_text("# c\nname=a\nrepo=b.db\n") + with pytest.raises(proton_pkg.PkgError, match="version"): + proton_pkg.read_lock(path) + + +def test_the_pinned_key_file_holds_the_pinned_fingerprint(tmp_path): + home = tmp_path / "g" + home.mkdir(mode=0o700) + try: + assert proton_pkg.key_fingerprints(proton_pkg.KEY, home) == [proton_pkg.FINGERPRINT] + finally: + stop_agent(home) + + +def test_package_urls_quote_the_epoch_colon(): + lock = proton_pkg.Lock("p", "https://m/repo/cachyos.db", "1:2-3", "x") + assert proton_pkg.package_url(lock, "p-1:2-3-x86_64.pkg.tar.zst") == ( + "https://m/repo/p-1%3A2-3-x86_64.pkg.tar.zst" + ) + + +def test_sha256_file_streams(tmp_path): + path = tmp_path / "f" + path.write_bytes(b"abc" * 100000) + assert proton_pkg.sha256_file(path) == hashlib.sha256(b"abc" * 100000).hexdigest() + + +def test_mtree_unescapes_octal_names(repo): + entries = proton_pkg.read_mtree(repo.package) + assert entries["user settings.py"].kind == "file" + assert entries["files/bin/msidb"].link == "wine" + assert ".INSTALL" not in entries and "" not in entries + assert PREFIX not in entries + + + + +def test_db_entries_other_than_desc_are_ignored(tmp_path): + db = tmp_path / "x.db" + with tarfile.open(db, "w:zst") as tar: + info = tarfile.TarInfo("p-1/files") + tar.addfile(info) + with pytest.raises(proton_pkg.PkgError, match="lists no p"): + proton_pkg.find_in_db(db, "p") + + +def test_a_key_file_without_the_pinned_fingerprint_is_refused(repo, tmp_path): + home = tmp_path / "g" + home.mkdir(mode=0o700) + try: + with pytest.raises(proton_pkg.PkgError, match="pinned key file"): + proton_pkg.verify_signature( + repo.package, repo.package, repo.key, "0" * 40, home + ) + finally: + stop_agent(home) + + +def test_a_package_without_mtree_is_refused(tmp_path): + package = tmp_path / "p.pkg.tar.zst" + with tarfile.open(package, "w:zst") as tar: + tar.addfile(tarfile.TarInfo("usr/x")) + with pytest.raises(proton_pkg.PkgError, match="no .MTREE"): + proton_pkg.read_mtree(package) + + +def test_directories_listed_in_mtree_need_no_content_check(tmp_path): + entries = {"files": proton_pkg.MtreeEntry("dir")} + (tmp_path / "files").mkdir() + assert proton_pkg.first_mismatch(tmp_path, entries) is None From c4ca111cd4616d75c8d44a928767b67cf2525fad Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 22 Nov 2026 15:02:55 +0100 Subject: [PATCH 20/24] Send a User-Agent; libibus without partial upgrade Co-Authored-By: Claude Opus 5.5 --- _installer_import/install.sh | 5 +++- .../steam_installer/proton_native.py | 4 +++- .../tests/test_proton_native_cli.py | 23 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/_installer_import/install.sh b/_installer_import/install.sh index ae43f40..f9fc334 100755 --- a/_installer_import/install.sh +++ b/_installer_import/install.sh @@ -60,8 +60,11 @@ ensure_libibus() { echo "missing: libibus -- run: sudo pacman -S --needed libibus" >&2 return 1 fi - sudo -n pacman -S --needed --noconfirm libibus || + sudo -n true 2>/dev/null || die "libibus is missing and sudo needs a password. Run: sudo pacman -S --needed libibus" + # No -y: syncing the db without upgrading is a partial upgrade on Arch. + sudo -n pacman -S --needed --noconfirm libibus || + die "pacman could not install libibus (a stale package db 404s). Run: sudo pacman -Syu libibus" } # Never runs the package's .INSTALL; see steam_installer/proton_native.py. diff --git a/_installer_import/steam_installer/proton_native.py b/_installer_import/steam_installer/proton_native.py index c8e42d1..14529af 100644 --- a/_installer_import/steam_installer/proton_native.py +++ b/_installer_import/steam_installer/proton_native.py @@ -55,7 +55,9 @@ def in_use(tool: Path) -> list[int]: def _download(url: str, dest: Path) -> None: - with urllib.request.urlopen(url, timeout=60) as response, dest.open("wb") as out: + # A generic agent: the mirror answers 403 to urllib's default one. + request = urllib.request.Request(url, headers={"User-Agent": "steam-game-installer"}) + with urllib.request.urlopen(request, timeout=60) as response, dest.open("wb") as out: shutil.copyfileobj(response, out, 1 << 20) diff --git a/_installer_import/tests/test_proton_native_cli.py b/_installer_import/tests/test_proton_native_cli.py index eae936d..e80dee7 100644 --- a/_installer_import/tests/test_proton_native_cli.py +++ b/_installer_import/tests/test_proton_native_cli.py @@ -56,3 +56,26 @@ def test_module_runs_as_a_script(tmp_path, coverage_subprocess_env): ) assert result.returncode == 0 assert "skipped" in result.stdout + + +def test_downloads_send_a_user_agent(monkeypatch, tmp_path): + """The CachyOS mirror answers 403 to urllib's default agent (2026-11-22).""" + import io + + seen = {} + + class _Response(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def fake_urlopen(request, timeout): + seen["agent"] = request.get_header("User-agent") + return _Response(b"data") + + monkeypatch.setattr(proton_native.urllib.request, "urlopen", fake_urlopen) + proton_native._download("https://m/x", tmp_path / "x") + assert (tmp_path / "x").read_bytes() == b"data" + assert seen["agent"] == "steam-game-installer" From b0a2113e10c3cb10be730b66e188bcd7aef29837 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 22 Nov 2026 15:10:42 +0100 Subject: [PATCH 21/24] Drop unused test imports Co-Authored-By: Claude Opus 5.5 --- _installer_import/tests/test_cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/_installer_import/tests/test_cli.py b/_installer_import/tests/test_cli.py index afd6e97..0d9a701 100644 --- a/_installer_import/tests/test_cli.py +++ b/_installer_import/tests/test_cli.py @@ -11,9 +11,7 @@ from steam_installer import ( appinfo, cli, depotdl, - manifest_write, recognize, - vdf, ) from _cli_helpers import ARGV, _stub_download From 4bab8f156d270efbf65f4e846a678bd0ca147533 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 22 Nov 2026 15:37:56 +0100 Subject: [PATCH 22/24] Freshness gate: fail closed on an old utils Co-Authored-By: Claude Opus 5.5 --- .../scripts/check_dependency_freshness.sh | 22 +++++++- .../tests/test_freshness_wrapper.py | 52 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 _installer_import/tests/test_freshness_wrapper.py diff --git a/_installer_import/scripts/check_dependency_freshness.sh b/_installer_import/scripts/check_dependency_freshness.sh index 49d6e29..641eeb7 100755 --- a/_installer_import/scripts/check_dependency_freshness.sh +++ b/_installer_import/scripts/check_dependency_freshness.sh @@ -18,7 +18,18 @@ set -euo pipefail -readonly SHARED_GATE="${UTILS_ROOT:-$HOME/src/utils}/scripts/check_dependency_freshness.sh" +readonly UTILS="${UTILS_ROOT:-$HOME/src/utils}" +readonly SHARED_GATE="$UTILS/scripts/check_dependency_freshness.sh" +readonly PACMAN_PARSER="$UTILS/dep_freshness/parsers/pacman_lock.py" + +# True when this run must judge the native-Proton lock (a full run, or the lock staged). +needs_pacman_adapter() { + local arg + for arg in "$@"; do + [[ "$arg" == "--all" || "$arg" == *proton-native.lock ]] && return 0 + done + return 1 +} main() { if [[ ! -x "$SHARED_GATE" ]]; then @@ -28,6 +39,15 @@ main() { exit 1 fi + # An older utils checkout skips a manifest it has no parser for and exits + # 0, which would wave any Proton pin through. Fail closed instead. + if needs_pacman_adapter "$@" && [[ ! -f "$PACMAN_PARSER" ]]; then + echo "Error: $UTILS has no dep_freshness/parsers/pacman_lock.py, so it" >&2 + echo " cannot check steam_installer/data/proton-native.lock." >&2 + echo " Update it: git -C $UTILS pull --ff-only" >&2 + exit 1 + fi + exec bash "$SHARED_GATE" "$@" } diff --git a/_installer_import/tests/test_freshness_wrapper.py b/_installer_import/tests/test_freshness_wrapper.py new file mode 100644 index 0000000..cdf3b6b --- /dev/null +++ b/_installer_import/tests/test_freshness_wrapper.py @@ -0,0 +1,52 @@ +"""The freshness wrapper fails closed when ~/src/utils cannot read the Proton lock.""" + +import subprocess +from pathlib import Path + +WRAPPER = Path(__file__).resolve().parent.parent / "scripts" / "check_dependency_freshness.sh" +LOCK = "steam_installer/data/proton-native.lock" + + +def _utils(tmp_path: Path, *, adapter: bool) -> Path: + root = tmp_path / "utils" + gate = root / "scripts" / "check_dependency_freshness.sh" + gate.parent.mkdir(parents=True) + gate.write_text("#!/bin/bash\necho shared-gate-ran\n") + gate.chmod(0o755) + if adapter: + parser = root / "dep_freshness" / "parsers" / "pacman_lock.py" + parser.parent.mkdir(parents=True) + parser.write_text("") + return root + + +def _run(root: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["bash", str(WRAPPER), *args], + env={"UTILS_ROOT": str(root), "PATH": "/usr/bin:/bin"}, + capture_output=True, + text=True, + check=False, + ) + + +def test_lock_without_the_pacman_adapter_fails(tmp_path): + result = _run(_utils(tmp_path, adapter=False), LOCK) + assert result.returncode == 1 + assert "pacman_lock" in result.stderr + assert "shared-gate-ran" not in result.stdout + + +def test_full_run_without_the_pacman_adapter_fails(tmp_path): + assert _run(_utils(tmp_path, adapter=False), "--all", "--strict").returncode == 1 + + +def test_lock_with_the_adapter_reaches_the_shared_gate(tmp_path): + result = _run(_utils(tmp_path, adapter=True), LOCK) + assert result.returncode == 0 + assert "shared-gate-ran" in result.stdout + + +def test_other_manifests_skip_the_adapter_check(tmp_path): + result = _run(_utils(tmp_path, adapter=False), "requirements.txt") + assert result.returncode == 0 From 96c5a1edef1a0368f83cd887d66e6a7126a88554 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 22 Nov 2026 15:58:01 +0100 Subject: [PATCH 23/24] Freshness gate: use utils origin/main snapshot Co-Authored-By: Claude Opus 5.5 --- .../scripts/check_dependency_freshness.sh | 47 +++++++++++++--- .../tests/test_freshness_wrapper.py | 53 +++++++++++++++++++ 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/_installer_import/scripts/check_dependency_freshness.sh b/_installer_import/scripts/check_dependency_freshness.sh index 641eeb7..934c6a5 100755 --- a/_installer_import/scripts/check_dependency_freshness.sh +++ b/_installer_import/scripts/check_dependency_freshness.sh @@ -31,6 +31,37 @@ needs_pacman_adapter() { return 1 } +# Print a gate directory that can read the Proton lock, or nothing. +# +# kuhy's ~/src/utils can lag origin/main (local commits not yet pushed, so it +# cannot fast-forward). Then the gate runs a read-only snapshot of utils' +# origin/main from the cache instead: the checkout itself is never touched. +gate_root_for_lock() { + if [[ -f "$PACMAN_PARSER" ]]; then + echo "$UTILS" + return + fi + local sha snapshot + local -a paths=(scripts/check_dependency_freshness.sh dep_freshness) + git -C "$UTILS" cat-file -e origin/main:dep_freshness/parsers/pacman_lock.py 2>/dev/null || + return 0 + sha="$(git -C "$UTILS" rev-parse origin/main)" + snapshot="${XDG_CACHE_HOME:-$HOME/.cache}/steam-game-installer/utils-gate/$sha" + if [[ ! -f "$snapshot/dep_freshness/parsers/pacman_lock.py" ]]; then + rm -rf "$snapshot.tmp" + mkdir -p "$snapshot.tmp" + # The fleet allowlist lives beside the gate, when utils has one. + if git -C "$UTILS" cat-file -e origin/main:dependency-freshness.allowlist.yaml 2>/dev/null; then + paths+=(dependency-freshness.allowlist.yaml) + fi + git -C "$UTILS" archive origin/main "${paths[@]}" | tar -x -C "$snapshot.tmp" + rm -rf "$snapshot" + mv "$snapshot.tmp" "$snapshot" + fi + echo "dependency-freshness: $UTILS lacks the pacman adapter; using its origin/main ${sha:0:12}" >&2 + echo "$snapshot" +} + main() { if [[ ! -x "$SHARED_GATE" ]]; then echo "Error: shared dependency-freshness gate not found at $SHARED_GATE" >&2 @@ -39,16 +70,20 @@ main() { exit 1 fi + local root="$UTILS" # An older utils checkout skips a manifest it has no parser for and exits # 0, which would wave any Proton pin through. Fail closed instead. - if needs_pacman_adapter "$@" && [[ ! -f "$PACMAN_PARSER" ]]; then - echo "Error: $UTILS has no dep_freshness/parsers/pacman_lock.py, so it" >&2 - echo " cannot check steam_installer/data/proton-native.lock." >&2 - echo " Update it: git -C $UTILS pull --ff-only" >&2 - exit 1 + if needs_pacman_adapter "$@"; then + root="$(gate_root_for_lock)" + if [[ -z "$root" ]]; then + echo "Error: $UTILS has no dep_freshness/parsers/pacman_lock.py, so it" >&2 + echo " cannot check steam_installer/data/proton-native.lock." >&2 + echo " Update it: git -C $UTILS pull --ff-only" >&2 + exit 1 + fi fi - exec bash "$SHARED_GATE" "$@" + exec bash "$root/scripts/check_dependency_freshness.sh" "$@" } main "$@" diff --git a/_installer_import/tests/test_freshness_wrapper.py b/_installer_import/tests/test_freshness_wrapper.py index cdf3b6b..755906e 100644 --- a/_installer_import/tests/test_freshness_wrapper.py +++ b/_installer_import/tests/test_freshness_wrapper.py @@ -50,3 +50,56 @@ def test_lock_with_the_adapter_reaches_the_shared_gate(tmp_path): def test_other_manifests_skip_the_adapter_check(tmp_path): result = _run(_utils(tmp_path, adapter=False), "requirements.txt") assert result.returncode == 0 + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True, check=True + ).stdout.strip() + + +def _diverged_utils(tmp_path: Path, *, origin_has_adapter: bool) -> Path: + """A utils checkout without the adapter whose origin/main may have it.""" + root = _utils(tmp_path, adapter=False) + _git(root, "init", "-q") + _git(root, "-c", "user.name=t", "-c", "user.email=t@t", "add", "-A") + _git(root, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "old") + old = _git(root, "rev-parse", "HEAD") + if origin_has_adapter: + gate = root / "scripts" / "check_dependency_freshness.sh" + gate.write_text('#!/bin/bash\necho "snapshot-gate-ran $(dirname "$0")"\n') + parser = root / "dep_freshness" / "parsers" / "pacman_lock.py" + parser.parent.mkdir(parents=True) + parser.write_text("") + _git(root, "-c", "user.name=t", "-c", "user.email=t@t", "add", "-A") + _git(root, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "new") + _git(root, "update-ref", "refs/remotes/origin/main", _git(root, "rev-parse", "HEAD")) + _git(root, "reset", "-q", "--hard", old) + return root + + +def _run_cached(root: Path, cache: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["bash", str(WRAPPER), *args], + env={"UTILS_ROOT": str(root), "PATH": "/usr/bin:/bin", "XDG_CACHE_HOME": str(cache)}, + capture_output=True, + text=True, + check=False, + ) + + +def test_old_checkout_runs_origin_main_snapshot(tmp_path): + root = _diverged_utils(tmp_path, origin_has_adapter=True) + result = _run_cached(root, tmp_path / "cache", LOCK) + assert result.returncode == 0, result.stderr + assert "snapshot-gate-ran" in result.stdout + assert str(tmp_path / "cache") in result.stdout + # The checkout itself is left exactly as it was. + assert not (root / "dep_freshness" / "parsers" / "pacman_lock.py").exists() + + +def test_old_checkout_and_old_origin_fail_closed(tmp_path): + root = _diverged_utils(tmp_path, origin_has_adapter=False) + result = _run_cached(root, tmp_path / "cache", LOCK) + assert result.returncode == 1 + assert "pacman_lock" in result.stderr From 889873f8dc89c412dc1de915feb7b8812d41ff05 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 22 Nov 2026 16:37:32 +0100 Subject: [PATCH 24/24] Zero all download counters in the manifest C18: settle_download_counters wrote equal pairs (1000/1000) from a broken manifest. Write all four counters as 0, like the from-scratch skeleton. Co-Authored-By: Claude Opus 5.5 --- .../steam_installer/manifest_write.py | 17 +++++------ _installer_import/tests/test_cli.py | 3 +- _installer_import/tests/test_cli_finish.py | 28 +++++++++++++++++++ .../tests/test_manifest_finish.py | 6 ++-- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/_installer_import/steam_installer/manifest_write.py b/_installer_import/steam_installer/manifest_write.py index bb00d3f..b21b769 100644 --- a/_installer_import/steam_installer/manifest_write.py +++ b/_installer_import/steam_installer/manifest_write.py @@ -213,16 +213,13 @@ def write_atomic(path: Path, text: str) -> None: def settle_download_counters(app_state: dict) -> None: - """Mark the download bookkeeping complete: `Downloaded` == `ToDownload`. + """Zero the download bookkeeping: nothing is queued, nothing is staged. Leaving these at their mid-download values is what made Steam discard a - StateFlags=4 manifest and resume fetching. Real installed manifests keep - these fields (they are not dropped) with the pair values equal. + StateFlags=4 manifest and resume fetching. All four go to "0", the same as + the from-scratch skeleton Steam accepted, rather than merely equal: the + older equal-pair form wrote `1000`/`1000` from a broken manifest, a value + no real install ever had. """ - for total_key, done_key in ( - ("BytesToDownload", "BytesDownloaded"), - ("BytesToStage", "BytesStaged"), - ): - total = app_state.get(total_key) - if total is not None: - app_state[done_key] = total + for key in ("BytesToDownload", "BytesDownloaded", "BytesToStage", "BytesStaged"): + app_state[key] = "0" diff --git a/_installer_import/tests/test_cli.py b/_installer_import/tests/test_cli.py index 0d9a701..6f209ea 100644 --- a/_installer_import/tests/test_cli.py +++ b/_installer_import/tests/test_cli.py @@ -29,7 +29,8 @@ def test_downloaded_content_finishes_the_manifest(steam, monkeypatch): app_state = recognize.read_app_state(library.steamapps / "appmanifest_1771300.acf") assert app_state["StateFlags"] == "4" - assert app_state["BytesDownloaded"] == app_state["BytesToDownload"] + for key in ("BytesToDownload", "BytesDownloaded", "BytesToStage", "BytesStaged"): + assert app_state[key] == "0", key assert recognize.download_counters_settled(app_state) diff --git a/_installer_import/tests/test_cli_finish.py b/_installer_import/tests/test_cli_finish.py index 2942006..f252cf7 100644 --- a/_installer_import/tests/test_cli_finish.py +++ b/_installer_import/tests/test_cli_finish.py @@ -198,3 +198,31 @@ def test_the_install_folder_is_corrected_even_when_the_download_fails( assert cli.main(ARGV) == 1 assert recognize.read_app_state(manifest)["installdir"] == "KingdomComeDeliverance2" + + +def test_wrong_installdir_and_mismatched_counters_end_correct_and_zeroed( + steam, monkeypatch +): + """C18: the live repro (installdir "... WRONG", BytesToDownload 1000, + BytesDownloaded 5) once came out as 1000/1000. The written manifest must + name the real folder, keep its files, and carry every counter at 0.""" + library, install_dir = steam + manifest = library.steamapps / "appmanifest_1771300.acf" + state = recognize.read_app_state(manifest) + state.update( + installdir="KingdomComeDeliverance2 WRONG", + BytesToDownload="1000", + BytesDownloaded="5", + BytesToStage="1000", + BytesStaged="5", + ) + manifest.write_text(vdf.dumps("AppState", state)) + _stub_download(monkeypatch) + + assert cli.main(ARGV) == 0 + + app_state = recognize.read_app_state(manifest) + assert app_state["installdir"] == "KingdomComeDeliverance2" + assert (install_dir / "game.pak").stat().st_size == 1000 + for key in ("BytesToDownload", "BytesDownloaded", "BytesToStage", "BytesStaged"): + assert app_state[key] == "0", key diff --git a/_installer_import/tests/test_manifest_finish.py b/_installer_import/tests/test_manifest_finish.py index 087e999..8d5774c 100644 --- a/_installer_import/tests/test_manifest_finish.py +++ b/_installer_import/tests/test_manifest_finish.py @@ -132,8 +132,10 @@ def test_finish_manifest_settles_download_counters(tmp_path): ) app_state = vdf.parse(manifest_path.read_text())["AppState"] - assert app_state["BytesDownloaded"] == app_state["BytesToDownload"] - assert app_state["BytesStaged"] == app_state["BytesToStage"] + # C18: all four counters at 0, as on a from-scratch manifest. Equal-but- + # nonzero leftovers (1000/1000) are not what Steam writes for an installed app. + for key in ("BytesToDownload", "BytesDownloaded", "BytesToStage", "BytesStaged"): + assert app_state[key] == "0", key assert app_state["StagingSize"] == "0" # DownloadType is present on every real installed manifest -- keep it. assert app_state["DownloadType"] == "1"