steam-backlog-enforcer/steam_backlog_enforcer/jobs/_spawn.py
Krzysztof kuhy Rudnicki 57d262b711
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (mypy, mypy, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (pylint, pylint, ) (push) Waiting to run
pre-commit / pre-commit (${{ matrix.group }}) (rest, , mypy,pylint) (push) Waiting to run
Tests / test (push) Waiting to run
Tests / web-coverage (push) Waiting to run
repo-contract / repo contract (push) Waiting to run
Pylint gate 10 over the whole package
Raise the pylint hook to --fail-under=10.0 and make it lint the whole
package on every run (pass_filenames: false): over the staged subset a
9.99 tree passed at commit time.

Ports de2aea3 (9b9342ba-1) onto main, minus its web/ and requirements.txt
bumps and the iptables dedupe f1822f1 already landed; _hltb_confidence and
_hltb_search keep main's 429 handling. _search_batch splits out
_acquire_auth / _search_headers, and _stats.py is back to 250 lines.

Drops the min-similarity-lines = 8 floor (0 duplicate-code hits at the
default) and sets max-attributes = 15 for the flat persisted records, as
kuhy decided. Tests stay outside pylint, as at every site already at 10.

Todo: fda71ed1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 16:57:35 +01:00

158 lines
5.5 KiB
Python

"""Start a job's detached process, and notice at once when it never started.
Under systemd the job runs through ``systemd-run --user --scope``. When that
cannot register the scope (no user bus, a bad ``XDG_RUNTIME_DIR``) it exits
straight away and the job never runs — yet ``Popen`` succeeded, so without a
check the job would sit ``queued`` until the dead-pid reconciler failed it
30 s later. :func:`spawn_job` waits the few milliseconds it takes to see
either the job's own interpreter (``systemd-run`` execs into it in place) or
the launcher's exit, and fails the job with the launcher's output instead.
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
from typing import BinaryIO, Final
from steam_backlog_enforcer.config import _atomic_write
from steam_backlog_enforcer.jobs._errors import JobError
from steam_backlog_enforcer.jobs._events import (
OUTPUT_NAME,
RESULT_NAME,
EventWriter,
read_events,
)
LOCK_FD_ENV: Final = "SBE_JOB_LOCK_FD"
_PACKAGE_ROOT = Path(__file__).resolve().parents[2]
# How long to wait for the launcher to exec into the job (or die). A healthy
# systemd-run takes milliseconds; past this the reconciler takes over.
_CONFIRM_SECONDS: Final = 3.0
_POLL_SECONDS: Final = 0.02
_OUTPUT_TAIL_CHARS: Final = 600
_EXIT_SPAWN_FAILED: Final = 1
def _spawn_argv(path: Path) -> list[str]:
"""The job subprocess command line.
Under systemd (the web server's user unit) the job is moved into its own
transient scope, so restarting the server's unit — which kills its
whole cgroup — does not take running jobs down with it.
"""
argv = [sys.executable, "-m", "steam_backlog_enforcer.jobs", "run", str(path)]
systemd_run = shutil.which("systemd-run")
if os.environ.get("INVOCATION_ID") and systemd_run:
unit = f"sbe-job-{path.name.lower()}"
argv = [systemd_run, "--user", "--scope", "--quiet", "--collect",
f"--unit={unit}", "--", *argv] # fmt: skip
return argv
def _is_job_interpreter(pid: int) -> bool:
"""Whether *pid* is already the job's Python (not still the launcher)."""
try:
argv0 = Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0", 1)[0]
except OSError:
return False
return argv0 == os.fsencode(sys.executable)
def _output_tail(path: Path) -> str:
"""The end of the job's captured stdout/stderr, for the failure log."""
try:
text = (path / OUTPUT_NAME).read_text(encoding="utf-8", errors="replace")
except OSError:
return ""
return text.strip()[-_OUTPUT_TAIL_CHARS:]
def _fail_unstarted(path: Path, summary: str) -> JobError:
"""Fail a job whose process never started; return the error to raise."""
tail = _output_tail(path)
writer = EventWriter(path)
writer.emit("log", level="error", message=f"{summary}\n{tail}".strip())
writer.emit("result", ok=False, summary=summary)
result = {"ok": False, "summary": summary, "state": "failed"}
_atomic_write(
path / RESULT_NAME,
json.dumps(result | {"exit_code": _EXIT_SPAWN_FAILED}, indent=2) + "\n",
)
writer.emit("state", state="failed")
return JobError(f"{summary} {tail}".strip(), code="op_failed")
def _confirm_started(proc: subprocess.Popen[bytes], path: Path) -> None:
"""Return once the job runs; raise if the launcher died before it did.
A job that already finished (fast read-only commands can, without
systemd) wrote its own result, so an exit with a result is a success.
Raises:
JobError: ``op_failed`` when the launcher exited without the job
ever running.
"""
deadline = time.monotonic() + _CONFIRM_SECONDS
while time.monotonic() < deadline:
code = proc.poll()
if code is not None:
if any(e.get("type") == "result" for e in read_events(path)):
return
msg = f"The job process could not be started (launcher exit {code})."
raise _fail_unstarted(path, msg)
if _is_job_interpreter(proc.pid):
return
time.sleep(_POLL_SECONDS)
def _start_session(
path: Path, out: BinaryIO, env: dict[str, str], lock_fd: int | None
) -> subprocess.Popen[bytes]:
"""Start the job process in its own session; the caller owns it.
No ``with``: the job outlives the CLI call that spawned it.
"""
return subprocess.Popen(
_spawn_argv(path),
stdin=subprocess.DEVNULL,
stdout=out,
stderr=out,
cwd=_PACKAGE_ROOT,
env=env,
start_new_session=True,
pass_fds=() if lock_fd is None else (lock_fd,),
)
def spawn_job(path: Path, lock_fd: int | None) -> int:
"""Start the detached job process for *path* and return its pid.
Args:
path: The prepared job directory.
lock_fd: The held one-mutating-job lock, handed to the child.
Raises:
JobError: ``op_failed`` when the process could not be started; the
job is already marked ``failed`` with the reason in its log.
"""
env = dict(os.environ)
env["PYTHONPATH"] = os.pathsep.join(
p for p in (str(_PACKAGE_ROOT), env.get("PYTHONPATH", "")) if p
)
if lock_fd is not None:
env[LOCK_FD_ENV] = str(lock_fd)
try:
with (path / OUTPUT_NAME).open("ab") as out:
proc = _start_session(path, out, env, lock_fd)
except OSError as exc:
msg = f"The job process could not be started: {exc}."
raise _fail_unstarted(path, msg) from exc
_confirm_started(proc, path)
return proc.pid