From 3acf913b323c1940be9fdbbcfbdb883b3e158018 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Fri, 18 Sep 2026 12:08:41 +0200 Subject: [PATCH] dep_freshness: pool every Maven repo; resolve each host once; retry EAI_AGAIN Three findings from proving the JitPack fix on the TachiyomiSY gate: - The first repository with a stable answer used to win. Central still carries com.github.ben-manes:gradle-versions-plugin 0.11.1 (2016) while the plugin portal has 0.54.0, so a 0.51.0 pin graded "current". Every repository is now asked and the newest stable across the pooled lists is the answer. - ~900 lookups, eight at a time, each resolving the registry host (A+AAAA) per connection: the LAN router's forwarder started dropping queries for every client on the LAN and the gate degraded itself. socket.getaddrinfo is memoised per (host, port, hints) for the run, and the per-host probe runs under a lock so eight workers make one probe, not eight. - A resolver EAI_AGAIN ("temporary failure") during the probe marked the registry unreachable for the whole run. The probe retries it HTTP_ATTEMPTS times; a permanent failure still fails immediately. Both retry loops lost their unreachable `return` sentinel (http.py is at 100% branch now); the resolver tests live in test_http_resolver.py to keep test_http.py under the 250-line cap. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016P4KZw1E2sXJjYqfcZWf1x --- dep_freshness/registries/http.py | 76 ++++++++++--- dep_freshness/registries/maven.py | 25 ++--- dep_freshness/tests/test_gradle_registry.py | 19 +++- dep_freshness/tests/test_http_resolver.py | 117 ++++++++++++++++++++ 4 files changed, 209 insertions(+), 28 deletions(-) create mode 100644 dep_freshness/tests/test_http_resolver.py diff --git a/dep_freshness/registries/http.py b/dep_freshness/registries/http.py index 09fa380..113a8d7 100644 --- a/dep_freshness/registries/http.py +++ b/dep_freshness/registries/http.py @@ -1,16 +1,22 @@ """The only module that touches the network. -Two things matter here. First, a single 2s per-host reachability probe +Three things matter here. First, a single 2s per-host reachability probe short-circuits every later request, so an offline run fails in seconds instead -of timing out once per package across ~900 lookups. Second, failures are -returned, never raised: the gate must degrade to a cached answer rather than -crash a pre-commit hook the user is forbidden from bypassing. +of timing out once per package across ~900 lookups. Second, each host is +resolved ONCE per run: `urlopen` asks the resolver (A and AAAA) on every +connection, and ~900 lookups eight at a time against six registry hosts sent +a burst the LAN router's DNS forwarder answered by dropping half of the +queries -- for every client on the LAN, for as long as the gate ran +(2026-09-18). Third, failures are returned, never raised: the gate must +degrade to a cached answer rather than crash a pre-commit hook the user is +forbidden from bypassing. """ from __future__ import annotations import json import socket +import threading from typing import Any from urllib.error import HTTPError, URLError from urllib.parse import urlsplit @@ -29,7 +35,30 @@ class Offline(Exception): _reachable: dict[str, bool] = {} +# Eight workers start on the same registry at once; without the lock each of +# them resolves and probes the host itself, and that burst alone (16 queries) +# was enough for the router's forwarder to drop some and mark Google Maven +# unreachable for the run. +_probe_lock = threading.Lock() _forced_offline = False +_addresses: dict[tuple[Any, ...], list[Any]] = {} +_system_getaddrinfo = socket.getaddrinfo + + +def cached_getaddrinfo(host: Any, port: Any, *args: Any) -> list[Any]: + """`socket.getaddrinfo`, memoised per (host, port, hints) for the run. + + `urllib` offers no resolver hook, so this replaces the socket-level + function for the process. Failures are not memoised: a host that did not + resolve is retried, and `host_reachable` remembers the verdict instead. + """ + key = (host, port, *args) + if key not in _addresses: + _addresses[key] = _system_getaddrinfo(host, port, *args) + return _addresses[key] + + +socket.getaddrinfo = cached_getaddrinfo def force_offline(value: bool = True) -> None: @@ -39,8 +68,9 @@ def force_offline(value: bool = True) -> None: def reset_probes() -> None: - """Forget cached reachability verdicts (tests, and `--refresh`).""" + """Forget cached reachability verdicts and addresses (tests, `--refresh`).""" _reachable.clear() + _addresses.clear() def host_reachable(url: str) -> bool: @@ -49,17 +79,34 @@ def host_reachable(url: str) -> bool: return False parts = urlsplit(url) host = parts.hostname or "" - if host in _reachable: - return _reachable[host] port = parts.port or (443 if parts.scheme == "https" else 80) - try: - with socket.create_connection((host, port), timeout=PROBE_TIMEOUT): - _reachable[host] = True - except OSError: - _reachable[host] = False + with _probe_lock: + if host not in _reachable: + _reachable[host] = _probe(host, port) return _reachable[host] +def _probe(host: str, port: int) -> bool: + """One TCP connect; a *temporary* resolver failure is retried. + + `EAI_AGAIN` is the resolver saying "ask again", not "no such host": a + LAN forwarder that drops a fifth of its queries produces it a few times + per run, and one of them used to mark a registry unreachable for the + whole run and degrade a strict gate to exit 3. + """ + attempt = 0 + while True: + attempt += 1 + try: + with socket.create_connection((host, port), timeout=PROBE_TIMEOUT): + return True + except socket.gaierror as exc: + if exc.errno != socket.EAI_AGAIN or attempt == HTTP_ATTEMPTS: + return False + except OSError: + return False + + def get_text(url: str, accept: str | None = None) -> str | None: """Fetch a body as text, or raise `Offline` / return None on 404. @@ -73,7 +120,9 @@ def get_text(url: str, accept: str | None = None) -> str | None: request = Request(url, headers=headers) # A slow registry is retried with a longer timeout each time; an HTTP # status is an answer and is never retried. - for attempt in range(1, HTTP_ATTEMPTS + 1): + attempt = 0 + while True: + attempt += 1 try: with urlopen(request, timeout=HTTP_TIMEOUT * attempt) as response: return response.read().decode("utf-8") @@ -84,7 +133,6 @@ def get_text(url: str, accept: str | None = None) -> str | None: except (URLError, TimeoutError, OSError, ValueError) as exc: if attempt == HTTP_ATTEMPTS: raise Offline(f"{url}: {exc}") from exc - return None # the loop always returns or raises; this satisfies the type def get_json(url: str, accept: str | None = None) -> Any: diff --git a/dep_freshness/registries/maven.py b/dep_freshness/registries/maven.py index bd1dbc6..b1cbb9a 100644 --- a/dep_freshness/registries/maven.py +++ b/dep_freshness/registries/maven.py @@ -1,10 +1,14 @@ """Maven repositories: newest stable of `group:artifact`, from maven-metadata. -Three repositories, asked in order until one knows the artifact: Google -Maven (every `androidx.*` and `com.android.*` artifact lives ONLY there), -Maven Central and the Gradle plugin portal (plugin marker artifacts -`:.gradle.plugin`). A 404 means "not here", not "not anywhere", so -it moves on; anything else surfaces as `Offline` from the HTTP layer. +Three repositories, ALL asked: Google Maven (every `androidx.*` and +`com.android.*` artifact lives ONLY there), Maven Central and the Gradle +plugin portal (plugin marker artifacts `:.gradle.plugin`). The +version lists are pooled and the newest stable across them wins. Stopping at +the first repository that knew the artifact graded +`com.github.ben-manes:gradle-versions-plugin` current at 0.51.0 because +Central still carries its 2016 releases (0.11.1) while the plugin portal has +0.54.0 (2026-09-18). A 404 means "not here", not "not anywhere"; anything +else surfaces as `Offline` from the HTTP layer. JitPack used to be the fourth: it served tag metadata for `com.github.*` artifacts published nowhere else (PhotoView, DirectionalViewPager). Since @@ -32,7 +36,7 @@ from dep_freshness._tables import GITHUB_REMOTE, MAVEN_REPOS from dep_freshness.registries._git import ls_remote from dep_freshness.registries.gitcommit import source_repo from dep_freshness.registries.http import get_text -from dep_freshness.versions import newest_per_generation, newest_stable, reference +from dep_freshness.versions import newest_per_generation, reference def metadata_url(repo: str, name: str) -> str | None: @@ -72,13 +76,8 @@ def latest(name: str) -> str | None: if url is None: return None body = get_text(url) - if body is None: - continue - versions = versions_in(body) - stable = newest_stable(versions) - if stable is not None: - return stable - seen.extend(versions) + if body is not None: + seen.extend(versions_in(body)) if not seen: seen.extend(github_tags(name)) return reference(seen) or newest_per_generation(seen) diff --git a/dep_freshness/tests/test_gradle_registry.py b/dep_freshness/tests/test_gradle_registry.py index 2ff7413..de02002 100644 --- a/dep_freshness/tests/test_gradle_registry.py +++ b/dep_freshness/tests/test_gradle_registry.py @@ -68,6 +68,15 @@ def test_a_repo_with_only_prereleases_does_not_stop_the_search(repos): assert maven.latest("g:a") == "0.9.0" +def test_every_repository_is_asked_and_the_newest_stable_across_them_wins(repos): + """gradle-versions-plugin: Central stopped at 0.11.1 in 2016, the plugin + portal carries 0.54.0. The first answer is not the answer.""" + name = "com.github.ben-manes:gradle-versions-plugin" + repos[maven.metadata_url(CENTRAL, name)] = metadata("0.11", "0.11.1") + repos[maven.metadata_url(PORTAL, name)] = metadata("0.53.0", "0.54.0") + assert maven.latest(name) == "0.54.0" + + def tags(monkeypatch, remote_tags: dict[str, list[str]]): """Serve `git ls-remote --tags --refs` for the GitHub remotes in `remote_tags`.""" calls: list[tuple[str, ...]] = [] @@ -100,7 +109,15 @@ def test_jitpack_artifact_is_answered_from_the_repo_tags(repos, monkeypatch): def test_a_project_named_tag_is_read_as_its_version(repos, monkeypatch): """`java-nat-sort` is tagged `natural-comparator-1.1`, not `1.1`.""" name = "com.github.gpanther:java-nat-sort" - tags(monkeypatch, {"gpanther/java-nat-sort": ["natural-comparator-1.0", "natural-comparator-1.1"]}) + tags( + monkeypatch, + { + "gpanther/java-nat-sort": [ + "natural-comparator-1.0", + "natural-comparator-1.1", + ] + }, + ) assert maven.latest(name) == "natural-comparator-1.1" diff --git a/dep_freshness/tests/test_http_resolver.py b/dep_freshness/tests/test_http_resolver.py new file mode 100644 index 0000000..2c97706 --- /dev/null +++ b/dep_freshness/tests/test_http_resolver.py @@ -0,0 +1,117 @@ +"""The resolver side of the network layer: one resolution and one probe per +host per run, and a *temporary* resolver failure that is retried rather than +declaring the registry unreachable. + +Written after the gate's own DNS burst (~900 lookups, eight at a time) made +the LAN router's forwarder drop queries for every client on the LAN +(2026-09-18). +""" + +from __future__ import annotations + +import io +import socket +import threading + +import pytest + +from dep_freshness.registries import http + + +@pytest.fixture(autouse=True) +def real_probe(monkeypatch): + """Undo conftest's blanket `host_reachable` stub for this module.""" + monkeypatch.undo() + http.reset_probes() + http.force_offline(False) + yield + http.reset_probes() + http.force_offline(False) + + +def test_each_host_is_resolved_once_per_run(monkeypatch): + """The process-wide resolver is the memoised one, and a repeat (host, + port, hints) key never reaches the system resolver again; `reset_probes` + forgets the addresses along with the reachability verdicts.""" + assert socket.getaddrinfo is http.cached_getaddrinfo + asked: list[tuple] = [] + + def system(host, port, *hints): + asked.append((host, port, *hints)) + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("192.0.2.1", port))] + + monkeypatch.setattr(http, "_system_getaddrinfo", system) + http.reset_probes() + first = socket.getaddrinfo("repo.example", 443, 0, socket.SOCK_STREAM) + assert socket.getaddrinfo("repo.example", 443, 0, socket.SOCK_STREAM) is first + socket.getaddrinfo("repo.example", 80, 0, socket.SOCK_STREAM) + assert [a[:2] for a in asked] == [("repo.example", 443), ("repo.example", 80)] + http.reset_probes() + socket.getaddrinfo("repo.example", 443, 0, socket.SOCK_STREAM) + assert len(asked) == 3 + + +def test_a_failed_resolution_is_not_remembered(monkeypatch): + def failing(host, port, *hints): + raise socket.gaierror(host) + + monkeypatch.setattr(http, "_system_getaddrinfo", failing) + http.reset_probes() + for _ in range(2): + with pytest.raises(socket.gaierror): + socket.getaddrinfo("gone.example", 443) + assert not http._addresses + + +def test_concurrent_workers_share_one_probe_per_host(monkeypatch): + """Eight threads asking about the same host at once produce one probe.""" + probes: list[str] = [] + gate = threading.Barrier(8) + + def fake_connect(address, timeout): + probes.append(address[0]) + return io.BytesIO() + + monkeypatch.setattr(socket, "create_connection", fake_connect) + + def worker(): + gate.wait() + assert http.host_reachable("https://repo.example/x") + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert probes == ["repo.example"] + + +def test_a_temporary_resolver_failure_is_retried_but_a_permanent_one_is_not( + monkeypatch, +): + outcomes = [socket.gaierror(socket.EAI_AGAIN, "try again"), io.BytesIO()] + + def flaky(address, timeout): + result = outcomes.pop(0) + if isinstance(result, Exception): + raise result + return result + + monkeypatch.setattr(socket, "create_connection", flaky) + assert http.host_reachable("https://flaky.example/x") + assert not outcomes + + def permanent(address, timeout): + raise socket.gaierror(socket.EAI_NONAME, "no such host") + + monkeypatch.setattr(socket, "create_connection", permanent) + assert not http.host_reachable("https://nonesuch.example/x") + + always_again = [socket.gaierror(socket.EAI_AGAIN, "again")] * http.HTTP_ATTEMPTS + + def exhausted(address, timeout): + raise always_again.pop(0) + + monkeypatch.setattr(socket, "create_connection", exhausted) + assert not http.host_reachable("https://never.example/x") + assert not always_again