mirror of
https://github.com/kuhyx/utils.git
synced 2026-11-22 11:12:26 +01:00
dep_freshness: retry a slow registry read with an escalating timeout before calling it offline
Some checks failed
dependency freshness / dependencies on newest stable (push) Has been cancelled
file length / file-length (push) Has been cancelled
md-naming / markdown naming convention (push) Has been cancelled
palette drift / palette (push) Has been cancelled
shared gate tests / gate unit tests (push) Has been cancelled
Some checks failed
dependency freshness / dependencies on newest stable (push) Has been cancelled
file length / file-length (push) Has been cancelled
md-naming / markdown naming convention (push) Has been cancelled
palette drift / palette (push) Has been cancelled
shared gate tests / gate unit tests (push) Has been cancelled
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
00bb6706fd
commit
78e92a7565
@ -89,6 +89,10 @@ CACHE_FILE: Final = "registry.json"
|
|||||||
TTL_SECONDS: Final = 6 * 3600
|
TTL_SECONDS: Final = 6 * 3600
|
||||||
TTL_GITTAG_SECONDS: Final = 24 * 3600 # also gitcommit: both cost a subprocess
|
TTL_GITTAG_SECONDS: Final = 24 * 3600 # also gitcommit: both cost a subprocess
|
||||||
HTTP_TIMEOUT: Final = 10.0
|
HTTP_TIMEOUT: Final = 10.0
|
||||||
|
# JitPack generates maven-metadata on request and routinely takes longer than
|
||||||
|
# the first timeout; one such read failed a whole CI gates run on 2026-09-12.
|
||||||
|
# Each retry waits `attempt * HTTP_TIMEOUT`, so three attempts allow a minute.
|
||||||
|
HTTP_ATTEMPTS: Final = 3
|
||||||
PROBE_TIMEOUT: Final = 2.0
|
PROBE_TIMEOUT: Final = 2.0
|
||||||
MAX_WORKERS: Final = 8
|
MAX_WORKERS: Final = 8
|
||||||
|
|
||||||
|
|||||||
@ -16,7 +16,12 @@ from urllib.error import HTTPError, URLError
|
|||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
from dep_freshness._tables import HTTP_TIMEOUT, PROBE_TIMEOUT, USER_AGENT
|
from dep_freshness._tables import (
|
||||||
|
HTTP_ATTEMPTS,
|
||||||
|
HTTP_TIMEOUT,
|
||||||
|
PROBE_TIMEOUT,
|
||||||
|
USER_AGENT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Offline(Exception):
|
class Offline(Exception):
|
||||||
@ -66,15 +71,20 @@ def get_text(url: str, accept: str | None = None) -> str | None:
|
|||||||
if accept:
|
if accept:
|
||||||
headers["Accept"] = accept
|
headers["Accept"] = accept
|
||||||
request = Request(url, headers=headers)
|
request = Request(url, headers=headers)
|
||||||
try:
|
# A slow registry is retried with a longer timeout each time; an HTTP
|
||||||
with urlopen(request, timeout=HTTP_TIMEOUT) as response:
|
# status is an answer and is never retried.
|
||||||
return response.read().decode("utf-8")
|
for attempt in range(1, HTTP_ATTEMPTS + 1):
|
||||||
except HTTPError as exc:
|
try:
|
||||||
if exc.code == 404:
|
with urlopen(request, timeout=HTTP_TIMEOUT * attempt) as response:
|
||||||
return None
|
return response.read().decode("utf-8")
|
||||||
raise Offline(f"{url}: HTTP {exc.code}") from exc
|
except HTTPError as exc:
|
||||||
except (URLError, TimeoutError, OSError, ValueError) as exc:
|
if exc.code == 404:
|
||||||
raise Offline(f"{url}: {exc}") from exc
|
return None
|
||||||
|
raise Offline(f"{url}: HTTP {exc.code}") from exc
|
||||||
|
except (URLError, TimeoutError, OSError, ValueError) as exc:
|
||||||
|
if attempt == HTTP_ATTEMPTS:
|
||||||
|
raise Offline(f"{url}: {exc}") from exc
|
||||||
|
return None # the loop always returns or raises; this satisfies the type
|
||||||
|
|
||||||
|
|
||||||
def get_json(url: str, accept: str | None = None) -> Any:
|
def get_json(url: str, accept: str | None = None) -> Any:
|
||||||
|
|||||||
@ -129,3 +129,34 @@ def test_get_text_returns_the_raw_body_and_none_on_404(monkeypatch):
|
|||||||
monkeypatch.setattr(http, "urlopen", _raiser(
|
monkeypatch.setattr(http, "urlopen", _raiser(
|
||||||
HTTPError("u", 404, "Not Found", {}, None)))
|
HTTPError("u", 404, "Not Found", {}, None)))
|
||||||
assert http.get_text("https://repo1.maven.org/x") is None
|
assert http.get_text("https://repo1.maven.org/x") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_slow_registry_is_retried_with_a_longer_timeout_each_time(monkeypatch):
|
||||||
|
"""JitPack builds maven-metadata on request; one 10 s timeout failed a
|
||||||
|
whole CI gates run. Timeouts escalate, an HTTP status never retries."""
|
||||||
|
_reachable(monkeypatch)
|
||||||
|
seen: list[float] = []
|
||||||
|
|
||||||
|
def slow_then_ok(_request, timeout):
|
||||||
|
seen.append(timeout)
|
||||||
|
if len(seen) < 3:
|
||||||
|
raise TimeoutError("timed out")
|
||||||
|
return _Response(b"<metadata/>")
|
||||||
|
|
||||||
|
monkeypatch.setattr(http, "urlopen", slow_then_ok)
|
||||||
|
assert http.get_text("https://jitpack.io/x/maven-metadata.xml") == "<metadata/>"
|
||||||
|
assert seen == [10.0, 20.0, 30.0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_registry_that_never_answers_is_offline_after_the_last_attempt(monkeypatch):
|
||||||
|
_reachable(monkeypatch)
|
||||||
|
calls: list[int] = []
|
||||||
|
|
||||||
|
def never(*_a, **_k):
|
||||||
|
calls.append(1)
|
||||||
|
raise TimeoutError("timed out")
|
||||||
|
|
||||||
|
monkeypatch.setattr(http, "urlopen", never)
|
||||||
|
with pytest.raises(http.Offline):
|
||||||
|
http.get_text("https://jitpack.io/x/maven-metadata.xml")
|
||||||
|
assert len(calls) == http.HTTP_ATTEMPTS
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user