diff --git a/dep_freshness/registries/http.py b/dep_freshness/registries/http.py index 09fa380..113a8d7 100644 --- a/dep_freshness/registries/http.py +++ b/dep_freshness/registries/http.py @@ -1,16 +1,22 @@ """The only module that touches the network. -Two things matter here. First, a single 2s per-host reachability probe +Three things matter here. First, a single 2s per-host reachability probe short-circuits every later request, so an offline run fails in seconds instead -of timing out once per package across ~900 lookups. Second, failures are -returned, never raised: the gate must degrade to a cached answer rather than -crash a pre-commit hook the user is forbidden from bypassing. +of timing out once per package across ~900 lookups. Second, each host is +resolved ONCE per run: `urlopen` asks the resolver (A and AAAA) on every +connection, and ~900 lookups eight at a time against six registry hosts sent +a burst the LAN router's DNS forwarder answered by dropping half of the +queries -- for every client on the LAN, for as long as the gate ran +(2026-09-18). Third, failures are returned, never raised: the gate must +degrade to a cached answer rather than crash a pre-commit hook the user is +forbidden from bypassing. """ from __future__ import annotations import json import socket +import threading from typing import Any from urllib.error import HTTPError, URLError from urllib.parse import urlsplit @@ -29,7 +35,30 @@ class Offline(Exception): _reachable: dict[str, bool] = {} +# Eight workers start on the same registry at once; without the lock each of +# them resolves and probes the host itself, and that burst alone (16 queries) +# was enough for the router's forwarder to drop some and mark Google Maven +# unreachable for the run. +_probe_lock = threading.Lock() _forced_offline = False +_addresses: dict[tuple[Any, ...], list[Any]] = {} +_system_getaddrinfo = socket.getaddrinfo + + +def cached_getaddrinfo(host: Any, port: Any, *args: Any) -> list[Any]: + """`socket.getaddrinfo`, memoised per (host, port, hints) for the run. + + `urllib` offers no resolver hook, so this replaces the socket-level + function for the process. Failures are not memoised: a host that did not + resolve is retried, and `host_reachable` remembers the verdict instead. + """ + key = (host, port, *args) + if key not in _addresses: + _addresses[key] = _system_getaddrinfo(host, port, *args) + return _addresses[key] + + +socket.getaddrinfo = cached_getaddrinfo def force_offline(value: bool = True) -> None: @@ -39,8 +68,9 @@ def force_offline(value: bool = True) -> None: def reset_probes() -> None: - """Forget cached reachability verdicts (tests, and `--refresh`).""" + """Forget cached reachability verdicts and addresses (tests, `--refresh`).""" _reachable.clear() + _addresses.clear() def host_reachable(url: str) -> bool: @@ -49,17 +79,34 @@ def host_reachable(url: str) -> bool: return False parts = urlsplit(url) host = parts.hostname or "" - if host in _reachable: - return _reachable[host] port = parts.port or (443 if parts.scheme == "https" else 80) - try: - with socket.create_connection((host, port), timeout=PROBE_TIMEOUT): - _reachable[host] = True - except OSError: - _reachable[host] = False + with _probe_lock: + if host not in _reachable: + _reachable[host] = _probe(host, port) return _reachable[host] +def _probe(host: str, port: int) -> bool: + """One TCP connect; a *temporary* resolver failure is retried. + + `EAI_AGAIN` is the resolver saying "ask again", not "no such host": a + LAN forwarder that drops a fifth of its queries produces it a few times + per run, and one of them used to mark a registry unreachable for the + whole run and degrade a strict gate to exit 3. + """ + attempt = 0 + while True: + attempt += 1 + try: + with socket.create_connection((host, port), timeout=PROBE_TIMEOUT): + return True + except socket.gaierror as exc: + if exc.errno != socket.EAI_AGAIN or attempt == HTTP_ATTEMPTS: + return False + except OSError: + return False + + def get_text(url: str, accept: str | None = None) -> str | None: """Fetch a body as text, or raise `Offline` / return None on 404. @@ -73,7 +120,9 @@ def get_text(url: str, accept: str | None = None) -> str | None: request = Request(url, headers=headers) # A slow registry is retried with a longer timeout each time; an HTTP # status is an answer and is never retried. - for attempt in range(1, HTTP_ATTEMPTS + 1): + attempt = 0 + while True: + attempt += 1 try: with urlopen(request, timeout=HTTP_TIMEOUT * attempt) as response: return response.read().decode("utf-8") @@ -84,7 +133,6 @@ def get_text(url: str, accept: str | None = None) -> str | None: except (URLError, TimeoutError, OSError, ValueError) as exc: if attempt == HTTP_ATTEMPTS: raise Offline(f"{url}: {exc}") from exc - return None # the loop always returns or raises; this satisfies the type def get_json(url: str, accept: str | None = None) -> Any: diff --git a/dep_freshness/registries/maven.py b/dep_freshness/registries/maven.py index bd1dbc6..b1cbb9a 100644 --- a/dep_freshness/registries/maven.py +++ b/dep_freshness/registries/maven.py @@ -1,10 +1,14 @@ """Maven repositories: newest stable of `group:artifact`, from maven-metadata. -Three repositories, asked in order until one knows the artifact: Google -Maven (every `androidx.*` and `com.android.*` artifact lives ONLY there), -Maven Central and the Gradle plugin portal (plugin marker artifacts -`:.gradle.plugin`). A 404 means "not here", not "not anywhere", so -it moves on; anything else surfaces as `Offline` from the HTTP layer. +Three repositories, ALL asked: Google Maven (every `androidx.*` and +`com.android.*` artifact lives ONLY there), Maven Central and the Gradle +plugin portal (plugin marker artifacts `:.gradle.plugin`). The +version lists are pooled and the newest stable across them wins. Stopping at +the first repository that knew the artifact graded +`com.github.ben-manes:gradle-versions-plugin` current at 0.51.0 because +Central still carries its 2016 releases (0.11.1) while the plugin portal has +0.54.0 (2026-09-18). A 404 means "not here", not "not anywhere"; anything +else surfaces as `Offline` from the HTTP layer. JitPack used to be the fourth: it served tag metadata for `com.github.*` artifacts published nowhere else (PhotoView, DirectionalViewPager). Since @@ -32,7 +36,7 @@ from dep_freshness._tables import GITHUB_REMOTE, MAVEN_REPOS from dep_freshness.registries._git import ls_remote from dep_freshness.registries.gitcommit import source_repo from dep_freshness.registries.http import get_text -from dep_freshness.versions import newest_per_generation, newest_stable, reference +from dep_freshness.versions import newest_per_generation, reference def metadata_url(repo: str, name: str) -> str | None: @@ -72,13 +76,8 @@ def latest(name: str) -> str | None: if url is None: return None body = get_text(url) - if body is None: - continue - versions = versions_in(body) - stable = newest_stable(versions) - if stable is not None: - return stable - seen.extend(versions) + if body is not None: + seen.extend(versions_in(body)) if not seen: seen.extend(github_tags(name)) return reference(seen) or newest_per_generation(seen) diff --git a/dep_freshness/tests/test_gradle_registry.py b/dep_freshness/tests/test_gradle_registry.py index 2ff7413..de02002 100644 --- a/dep_freshness/tests/test_gradle_registry.py +++ b/dep_freshness/tests/test_gradle_registry.py @@ -68,6 +68,15 @@ def test_a_repo_with_only_prereleases_does_not_stop_the_search(repos): assert maven.latest("g:a") == "0.9.0" +def test_every_repository_is_asked_and_the_newest_stable_across_them_wins(repos): + """gradle-versions-plugin: Central stopped at 0.11.1 in 2016, the plugin + portal carries 0.54.0. The first answer is not the answer.""" + name = "com.github.ben-manes:gradle-versions-plugin" + repos[maven.metadata_url(CENTRAL, name)] = metadata("0.11", "0.11.1") + repos[maven.metadata_url(PORTAL, name)] = metadata("0.53.0", "0.54.0") + assert maven.latest(name) == "0.54.0" + + def tags(monkeypatch, remote_tags: dict[str, list[str]]): """Serve `git ls-remote --tags --refs` for the GitHub remotes in `remote_tags`.""" calls: list[tuple[str, ...]] = [] @@ -100,7 +109,15 @@ def test_jitpack_artifact_is_answered_from_the_repo_tags(repos, monkeypatch): def test_a_project_named_tag_is_read_as_its_version(repos, monkeypatch): """`java-nat-sort` is tagged `natural-comparator-1.1`, not `1.1`.""" name = "com.github.gpanther:java-nat-sort" - tags(monkeypatch, {"gpanther/java-nat-sort": ["natural-comparator-1.0", "natural-comparator-1.1"]}) + tags( + monkeypatch, + { + "gpanther/java-nat-sort": [ + "natural-comparator-1.0", + "natural-comparator-1.1", + ] + }, + ) assert maven.latest(name) == "natural-comparator-1.1" diff --git a/dep_freshness/tests/test_http_resolver.py b/dep_freshness/tests/test_http_resolver.py new file mode 100644 index 0000000..2c97706 --- /dev/null +++ b/dep_freshness/tests/test_http_resolver.py @@ -0,0 +1,117 @@ +"""The resolver side of the network layer: one resolution and one probe per +host per run, and a *temporary* resolver failure that is retried rather than +declaring the registry unreachable. + +Written after the gate's own DNS burst (~900 lookups, eight at a time) made +the LAN router's forwarder drop queries for every client on the LAN +(2026-09-18). +""" + +from __future__ import annotations + +import io +import socket +import threading + +import pytest + +from dep_freshness.registries import http + + +@pytest.fixture(autouse=True) +def real_probe(monkeypatch): + """Undo conftest's blanket `host_reachable` stub for this module.""" + monkeypatch.undo() + http.reset_probes() + http.force_offline(False) + yield + http.reset_probes() + http.force_offline(False) + + +def test_each_host_is_resolved_once_per_run(monkeypatch): + """The process-wide resolver is the memoised one, and a repeat (host, + port, hints) key never reaches the system resolver again; `reset_probes` + forgets the addresses along with the reachability verdicts.""" + assert socket.getaddrinfo is http.cached_getaddrinfo + asked: list[tuple] = [] + + def system(host, port, *hints): + asked.append((host, port, *hints)) + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("192.0.2.1", port))] + + monkeypatch.setattr(http, "_system_getaddrinfo", system) + http.reset_probes() + first = socket.getaddrinfo("repo.example", 443, 0, socket.SOCK_STREAM) + assert socket.getaddrinfo("repo.example", 443, 0, socket.SOCK_STREAM) is first + socket.getaddrinfo("repo.example", 80, 0, socket.SOCK_STREAM) + assert [a[:2] for a in asked] == [("repo.example", 443), ("repo.example", 80)] + http.reset_probes() + socket.getaddrinfo("repo.example", 443, 0, socket.SOCK_STREAM) + assert len(asked) == 3 + + +def test_a_failed_resolution_is_not_remembered(monkeypatch): + def failing(host, port, *hints): + raise socket.gaierror(host) + + monkeypatch.setattr(http, "_system_getaddrinfo", failing) + http.reset_probes() + for _ in range(2): + with pytest.raises(socket.gaierror): + socket.getaddrinfo("gone.example", 443) + assert not http._addresses + + +def test_concurrent_workers_share_one_probe_per_host(monkeypatch): + """Eight threads asking about the same host at once produce one probe.""" + probes: list[str] = [] + gate = threading.Barrier(8) + + def fake_connect(address, timeout): + probes.append(address[0]) + return io.BytesIO() + + monkeypatch.setattr(socket, "create_connection", fake_connect) + + def worker(): + gate.wait() + assert http.host_reachable("https://repo.example/x") + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert probes == ["repo.example"] + + +def test_a_temporary_resolver_failure_is_retried_but_a_permanent_one_is_not( + monkeypatch, +): + outcomes = [socket.gaierror(socket.EAI_AGAIN, "try again"), io.BytesIO()] + + def flaky(address, timeout): + result = outcomes.pop(0) + if isinstance(result, Exception): + raise result + return result + + monkeypatch.setattr(socket, "create_connection", flaky) + assert http.host_reachable("https://flaky.example/x") + assert not outcomes + + def permanent(address, timeout): + raise socket.gaierror(socket.EAI_NONAME, "no such host") + + monkeypatch.setattr(socket, "create_connection", permanent) + assert not http.host_reachable("https://nonesuch.example/x") + + always_again = [socket.gaierror(socket.EAI_AGAIN, "again")] * http.HTTP_ATTEMPTS + + def exhausted(address, timeout): + raise always_again.pop(0) + + monkeypatch.setattr(socket, "create_connection", exhausted) + assert not http.host_reachable("https://never.example/x") + assert not always_again