dep_freshness: retry a slow registry read with an escalating timeout before calling it offline

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

dep_freshness/_tables.py

@@ -89,6 +89,10 @@ CACHE_FILE: Final = "registry.json"
 TTL_SECONDS: Final = 6 * 3600
 TTL_GITTAG_SECONDS: Final = 24 * 3600  # also gitcommit: both cost a subprocess
 HTTP_TIMEOUT: Final = 10.0
+# JitPack generates maven-metadata on request and routinely takes longer than
+# the first timeout; one such read failed a whole CI gates run on 2026-09-12.
+# Each retry waits `attempt * HTTP_TIMEOUT`, so three attempts allow a minute.
+HTTP_ATTEMPTS: Final = 3
 PROBE_TIMEOUT: Final = 2.0
 MAX_WORKERS: Final = 8
 

dep_freshness/registries/http.py

@@ -16,7 +16,12 @@ from urllib.error import HTTPError, URLError
 from urllib.parse import urlsplit
 from urllib.request import Request, urlopen
 
-from dep_freshness._tables import HTTP_TIMEOUT, PROBE_TIMEOUT, USER_AGENT
+from dep_freshness._tables import (
+    HTTP_ATTEMPTS,
+    HTTP_TIMEOUT,
+    PROBE_TIMEOUT,
+    USER_AGENT,
+)
 
 
 class Offline(Exception):
@@ -66,15 +71,20 @@ def get_text(url: str, accept: str | None = None) -> str | None:
     if accept:
         headers["Accept"] = accept
     request = Request(url, headers=headers)
-    try:
-        with urlopen(request, timeout=HTTP_TIMEOUT) as response:
-            return response.read().decode("utf-8")
-    except HTTPError as exc:
-        if exc.code == 404:
-            return None
-        raise Offline(f"{url}: HTTP {exc.code}") from exc
-    except (URLError, TimeoutError, OSError, ValueError) as exc:
-        raise Offline(f"{url}: {exc}") from exc
+    # A slow registry is retried with a longer timeout each time; an HTTP
+    # status is an answer and is never retried.
+    for attempt in range(1, HTTP_ATTEMPTS + 1):
+        try:
+            with urlopen(request, timeout=HTTP_TIMEOUT * attempt) as response:
+                return response.read().decode("utf-8")
+        except HTTPError as exc:
+            if exc.code == 404:
+                return None
+            raise Offline(f"{url}: HTTP {exc.code}") from exc
+        except (URLError, TimeoutError, OSError, ValueError) as exc:
+            if attempt == HTTP_ATTEMPTS:
+                raise Offline(f"{url}: {exc}") from exc
+    return None  # the loop always returns or raises; this satisfies the type
 
 
 def get_json(url: str, accept: str | None = None) -> Any:

dep_freshness/tests/test_http.py

@@ -129,3 +129,34 @@ def test_get_text_returns_the_raw_body_and_none_on_404(monkeypatch):
     monkeypatch.setattr(http, "urlopen", _raiser(
         HTTPError("u", 404, "Not Found", {}, None)))
     assert http.get_text("https://repo1.maven.org/x") is None
+
+
+def test_a_slow_registry_is_retried_with_a_longer_timeout_each_time(monkeypatch):
+    """JitPack builds maven-metadata on request; one 10 s timeout failed a
+    whole CI gates run. Timeouts escalate, an HTTP status never retries."""
+    _reachable(monkeypatch)
+    seen: list[float] = []
+
+    def slow_then_ok(_request, timeout):
+        seen.append(timeout)
+        if len(seen) < 3:
+            raise TimeoutError("timed out")
+        return _Response(b"<metadata/>")
+
+    monkeypatch.setattr(http, "urlopen", slow_then_ok)
+    assert http.get_text("https://jitpack.io/x/maven-metadata.xml") == "<metadata/>"
+    assert seen == [10.0, 20.0, 30.0]
+
+
+def test_a_registry_that_never_answers_is_offline_after_the_last_attempt(monkeypatch):
+    _reachable(monkeypatch)
+    calls: list[int] = []
+
+    def never(*_a, **_k):
+        calls.append(1)
+        raise TimeoutError("timed out")
+
+    monkeypatch.setattr(http, "urlopen", never)
+    with pytest.raises(http.Offline):
+        http.get_text("https://jitpack.io/x/maven-metadata.xml")
+    assert len(calls) == http.HTTP_ATTEMPTS