diff --git a/dep_freshness/_tables.py b/dep_freshness/_tables.py index 3137c2f..79a75f0 100644 --- a/dep_freshness/_tables.py +++ b/dep_freshness/_tables.py @@ -89,6 +89,10 @@ CACHE_FILE: Final = "registry.json" TTL_SECONDS: Final = 6 * 3600 TTL_GITTAG_SECONDS: Final = 24 * 3600 # also gitcommit: both cost a subprocess HTTP_TIMEOUT: Final = 10.0 +# JitPack generates maven-metadata on request and routinely takes longer than +# the first timeout; one such read failed a whole CI gates run on 2026-09-12. +# Each retry waits `attempt * HTTP_TIMEOUT`, so three attempts allow a minute. +HTTP_ATTEMPTS: Final = 3 PROBE_TIMEOUT: Final = 2.0 MAX_WORKERS: Final = 8 diff --git a/dep_freshness/registries/http.py b/dep_freshness/registries/http.py index 2f60c96..09fa380 100644 --- a/dep_freshness/registries/http.py +++ b/dep_freshness/registries/http.py @@ -16,7 +16,12 @@ from urllib.error import HTTPError, URLError from urllib.parse import urlsplit from urllib.request import Request, urlopen -from dep_freshness._tables import HTTP_TIMEOUT, PROBE_TIMEOUT, USER_AGENT +from dep_freshness._tables import ( + HTTP_ATTEMPTS, + HTTP_TIMEOUT, + PROBE_TIMEOUT, + USER_AGENT, +) class Offline(Exception): @@ -66,15 +71,20 @@ def get_text(url: str, accept: str | None = None) -> str | None: if accept: headers["Accept"] = accept request = Request(url, headers=headers) - try: - with urlopen(request, timeout=HTTP_TIMEOUT) as response: - return response.read().decode("utf-8") - except HTTPError as exc: - if exc.code == 404: - return None - raise Offline(f"{url}: HTTP {exc.code}") from exc - except (URLError, TimeoutError, OSError, ValueError) as exc: - raise Offline(f"{url}: {exc}") from exc + # A slow registry is retried with a longer timeout each time; an HTTP + # status is an answer and is never retried. + for attempt in range(1, HTTP_ATTEMPTS + 1): + try: + with urlopen(request, timeout=HTTP_TIMEOUT * attempt) as response: + return response.read().decode("utf-8") + except HTTPError as exc: + if exc.code == 404: + return None + raise Offline(f"{url}: HTTP {exc.code}") from exc + except (URLError, TimeoutError, OSError, ValueError) as exc: + if attempt == HTTP_ATTEMPTS: + raise Offline(f"{url}: {exc}") from exc + return None # the loop always returns or raises; this satisfies the type def get_json(url: str, accept: str | None = None) -> Any: diff --git a/dep_freshness/tests/test_http.py b/dep_freshness/tests/test_http.py index 47b3cf4..3132d44 100644 --- a/dep_freshness/tests/test_http.py +++ b/dep_freshness/tests/test_http.py @@ -129,3 +129,34 @@ def test_get_text_returns_the_raw_body_and_none_on_404(monkeypatch): monkeypatch.setattr(http, "urlopen", _raiser( HTTPError("u", 404, "Not Found", {}, None))) assert http.get_text("https://repo1.maven.org/x") is None + + +def test_a_slow_registry_is_retried_with_a_longer_timeout_each_time(monkeypatch): + """JitPack builds maven-metadata on request; one 10 s timeout failed a + whole CI gates run. Timeouts escalate, an HTTP status never retries.""" + _reachable(monkeypatch) + seen: list[float] = [] + + def slow_then_ok(_request, timeout): + seen.append(timeout) + if len(seen) < 3: + raise TimeoutError("timed out") + return _Response(b"") + + monkeypatch.setattr(http, "urlopen", slow_then_ok) + assert http.get_text("https://jitpack.io/x/maven-metadata.xml") == "" + assert seen == [10.0, 20.0, 30.0] + + +def test_a_registry_that_never_answers_is_offline_after_the_last_attempt(monkeypatch): + _reachable(monkeypatch) + calls: list[int] = [] + + def never(*_a, **_k): + calls.append(1) + raise TimeoutError("timed out") + + monkeypatch.setattr(http, "urlopen", never) + with pytest.raises(http.Offline): + http.get_text("https://jitpack.io/x/maven-metadata.xml") + assert len(calls) == http.HTTP_ATTEMPTS