dep_freshness: upstream: allowlist predicate for forks; Google API clients judged inside their generation
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
file length / file-length (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
palette drift / palette (push) Waiting to run
shared gate tests / gate unit tests (push) Waiting to run

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Krzysztof Rudnicki 2026-09-12 20:32:34 +02:00
parent 94f6873148
commit 4d00d6ccde
11 changed files with 412 additions and 23 deletions

View File

@ -195,3 +195,9 @@ ALLOWLIST_FILE: Final = "dependency-freshness.allowlist.yaml"
SHARED_ALLOWLIST_ENV: Final = "DEP_FRESHNESS_SHARED_ALLOWLIST"
ALLOWLIST_MAX_DAYS: Final = 90
TRANSITIVE_PREFIX: Final = "transitive:"
# A fork's predicate: the entry holds only while the upstream repository's own
# copy of the same manifest pins the same version, and dies the day upstream
# bumps. TachiyomiSY cannot take moko-resources 0.27.0 (a 300-file accessor
# rewrite that breaks every future upstream string) before jobobby04 does.
UPSTREAM_PREFIX: Final = "upstream:"
GITHUB_RAW: Final = "https://raw.githubusercontent.com/{repo}/HEAD/{path}"

View File

@ -6,6 +6,9 @@ Two entry classes, and the distinction is the whole point:
re-evaluated every run and the entry clears itself when upstream moves. It
never expires on a calendar, because a date nobody can act on turns the gate
red for a reason no commit can fix.
* `blocked_by: upstream:<owner>/<repo>` is the fork's predicate: it holds
while that repository's copy of the same manifest pins the same version
(`upstream.py`), and dies the day upstream bumps.
* anything else is **discretionary**: it needs `expires`, capped at 90 days,
and fails once past.
@ -15,13 +18,12 @@ allowlist that rots silently is a suppression, not an exception.
from __future__ import annotations
import os
from datetime import date, datetime
from pathlib import Path
import yaml
import os
from dep_freshness._tables import (
ALLOWLIST_FILE,
ALLOWLIST_MAX_DAYS,
@ -79,17 +81,19 @@ def _entry(raw: object, source: Path, position: int) -> Exception_:
expires=(str(raw["expires"]).strip() if raw.get("expires") else None),
source=source,
)
if entry.transitive:
if entry.predicate:
if entry.expires:
raise AllowlistError(
f"{where}: a transitive: entry must NOT set expires -- it clears "
f"{where}: a predicate entry must NOT set expires -- it clears "
"itself when the blocker lifts, and a date nobody can act on "
"only turns the gate red for no actionable reason"
)
if not entry.blocker or not all(entry.blocker):
if entry.transitive and (not entry.blocker or not all(entry.blocker)):
raise AllowlistError(
f"{where}: blocked_by must read transitive:<package>@<version>"
)
if entry.upstream and len((entry.upstream_repo or "").split("/")) != 2:
raise AllowlistError(f"{where}: blocked_by must read upstream:<owner>/<repo>")
return entry
if not entry.expires:
raise AllowlistError(f"{where}: a discretionary entry requires expires")
@ -98,7 +102,7 @@ def _entry(raw: object, source: Path, position: int) -> Exception_:
def check_expiry(entry: Exception_, today: date | None = None) -> None:
"""Raise if a discretionary entry is past due or reaches too far out."""
if entry.transitive or not entry.expires:
if entry.predicate or not entry.expires:
return
now = today or date.today()
when = _parse_date(entry.expires, f"{entry.source}: {entry.package}")

View File

@ -27,7 +27,9 @@ from dep_freshness.evaluate import judge
from dep_freshness.models import Finding, Severity
from dep_freshness.quarantine import installable_latest
from dep_freshness.registries import http
from dep_freshness.registries.http import Offline
from dep_freshness.resolve import Answer, Resolver
from dep_freshness.upstream import still_pins
def repo_root(start: Path) -> Path:
@ -45,8 +47,9 @@ def repo_root(start: Path) -> Path:
def _excuse(
findings: list[Finding], entries, shared: Path | None = None,
whole_repo: bool = True,
) -> tuple[list[Finding], list]:
whole_repo: bool = True, root: Path | None = None,
degraded: list[str] | None = None,
) -> tuple[list[Finding], list, dict[str, bool]]:
"""Split findings into (still failing, excused), and flag dead entries.
A `transitive:` entry clears itself the moment its dependency is no longer
@ -68,6 +71,7 @@ def _excuse(
"""
by_label = {f"{e.ecosystem}:{e.package}": e for e in entries}
used: set[str] = set()
blocking: dict[str, bool] = {}
failing: list[Finding] = []
excused: list[Finding] = []
for finding in findings:
@ -76,9 +80,18 @@ def _excuse(
used.add(finding.label) # the entry applies here, even if it
# does not excuse this particular pin -- that is a stale pin to
# fix, not allowlist rot to delete.
blocking[finding.label] = True
if entry is None or not _covers(entry, finding):
failing.append(finding)
continue
if entry.upstream and not _upstream_holds(entry, finding, root, degraded):
blocking[finding.label] = False
failing.append(Finding(
finding.dep, finding.severity, finding.latest,
f"{entry.upstream_repo} no longer pins {entry.pinned}: drop the "
"allowlist entry and take the bump",
))
continue
excused.append(Finding(
finding.dep, finding.severity, finding.latest, finding.detail,
excused=entry.reason,
@ -89,7 +102,17 @@ def _excuse(
and f"{e.ecosystem}:{e.package}" not in used
and (shared is None or e.source != shared)
]
return failing, dead
return failing, dead, blocking
def _upstream_holds(entry, finding: Finding, root, degraded) -> bool:
"""Offline, an `upstream:` entry is trusted and the run is marked degraded."""
try:
return still_pins(entry, finding, root or Path.cwd())
except Offline as exc:
if degraded is not None:
degraded.append(f"{entry.blocked_by} unverified: {exc}")
return True
def _covers(entry, finding: Finding) -> bool:
@ -187,13 +210,11 @@ def main(argv: list[str] | None = None) -> int:
resolver = Resolver(refresh=args.refresh)
findings = collect(_targets(args, root), resolver)
shared = shared_path()
failing, dead = _excuse(
failing, dead, still_blocking = _excuse(
findings, entries,
shared=None if root == shared.parent else shared,
whole_repo=args.all,
whole_repo=args.all, root=root, degraded=resolver.degraded,
)
still_blocking = {f.label: True for f in findings}
report.exceptions_block(entries, still_blocking)
unknown = [f for f in failing if f.severity is Severity.UNKNOWN]

View File

@ -19,7 +19,7 @@ from dep_freshness._tables import GITCOMMIT, MATRIX, TOOLCHAIN
from dep_freshness.constraints import lower_bound, satisfies
from dep_freshness.models import Dep, Finding, Severity
from dep_freshness.resolve import Answer
from dep_freshness.versions import behind
from dep_freshness.versions import behind, google_api_key
# What JitPack accepts as a version, and what a bump pastes into the catalog.
_SHORT_SHA = 10
@ -46,6 +46,24 @@ def _commit(dep: Dep, heads: str) -> Finding | None:
)
def _generation(dep: Dep, pin, latest: str) -> Finding | None:
"""A Google API client is judged inside its own generation only."""
gen, rank = pin
for candidate in latest.split():
key = google_api_key(candidate)
if key is None or key[0] != gen:
continue
if key[1] > rank:
return Finding(dep, Severity.STALE, candidate)
return None
return Finding(
dep,
Severity.UNKNOWN,
None,
detail=f"the registry has no {gen} generation of this client any more",
)
def _toolchain(dep: Dep, latest: str) -> Finding | None:
if dep.pinned:
if behind(dep.pinned, latest):
@ -112,6 +130,9 @@ def judge(dep: Dep, answer: Answer) -> Finding | None:
return _toolchain(dep, latest)
if dep.ecosystem == GITCOMMIT:
return _commit(dep, latest)
pin = google_api_key(dep.pinned or "")
if pin is not None:
return _generation(dep, pin, latest)
if dep.peer:
# A peerDependency declares what a CONSUMER may bring, so exact-pinning

View File

@ -76,11 +76,32 @@ class Exception_:
@property
def transitive(self) -> bool:
"""True when this is a predicate entry that never expires on a date."""
"""True when a named transitive dependency is what holds the pin."""
from dep_freshness._tables import TRANSITIVE_PREFIX
return self.blocked_by.startswith(TRANSITIVE_PREFIX)
@property
def upstream(self) -> bool:
"""True when the fork's upstream still pinning this version holds it."""
from dep_freshness._tables import UPSTREAM_PREFIX
return self.blocked_by.startswith(UPSTREAM_PREFIX)
@property
def predicate(self) -> bool:
"""True for either self-clearing class; False means a calendar expiry."""
return self.transitive or self.upstream
@property
def upstream_repo(self) -> str | None:
"""`owner/repo` of the upstream an `upstream:` entry names."""
from dep_freshness._tables import UPSTREAM_PREFIX
if not self.upstream:
return None
return self.blocked_by[len(UPSTREAM_PREFIX):].strip()
@property
def blocker(self) -> tuple[str, str] | None:
"""`(package, version)` this entry claims is holding the pin back."""

View File

@ -13,7 +13,9 @@ layer.
through `newest_stable` like every other ecosystem. Only when NO repository
has a stable release at all does the newest pre-release become the reference
(`versions.reference`), which is how `androidx.biometric:biometric-ktx`,
alpha-only since 2021, gets a checkable answer.
alpha-only since 2021, gets a checkable answer. Google's API clients
(`v3-rev20260901-2.0.0`) are neither: for those the answer is the newest
revision of every generation, and the judge picks the pin's own.
"""
from __future__ import annotations
@ -22,7 +24,7 @@ from xml.etree import ElementTree
from dep_freshness._tables import MAVEN_REPOS
from dep_freshness.registries.http import get_text
from dep_freshness.versions import newest_stable, reference
from dep_freshness.versions import newest_per_generation, newest_stable, reference
def metadata_url(repo: str, name: str) -> str | None:
@ -60,4 +62,4 @@ def latest(name: str) -> str | None:
if stable is not None:
return stable
seen.extend(versions)
return reference(seen)
return reference(seen) or newest_per_generation(seen)

View File

@ -7,13 +7,14 @@ allowlist is that every not-latest pin stays visible.
from __future__ import annotations
from datetime import date, datetime
import json
from pathlib import Path
import sys
from datetime import date, datetime
from pathlib import Path
from dep_freshness.models import Exception_, Finding, Severity
def _err(stream):
"""Resolve stderr at call time.
@ -58,7 +59,7 @@ def exceptions_block(
print(f"\n⚠️ DEPENDENCY EXCEPTION IN USE — {len(entries)} active", file=stream)
for entry in entries:
print(f" {entry.package} {entry.pinned} ({entry.ecosystem})", file=stream)
if entry.transitive:
if entry.predicate:
state = "still blocking" if still_blocking.get(
f"{entry.ecosystem}:{entry.package}", True) else "CLEARED"
print(f" blocked_by: {entry.blocked_by} [{state}]", file=stream)
@ -76,7 +77,7 @@ def machine_lines(entries: list[Exception_]) -> list[str]:
head = (f"[DEP-EXCEPTION] {entry.ecosystem}:{entry.package} {entry.pinned}")
if entry.latest_known:
head += f" < {entry.latest_known}"
if entry.transitive:
if entry.predicate:
out.append(f"{head} blocked_by={entry.blocked_by} [still blocking]")
else:
left = _days_left(entry.expires or "")
@ -107,7 +108,7 @@ def violations(findings: list[Finding], root: Path, stream=None) -> None:
)
if finding.detail:
print(f" {finding.detail}", file=stream)
print("", file=stream)
print(file=stream)
print(
"Bump them to latest stable, or record an exception in "
"dependency-freshness.allowlist.yaml with reason + blocked_by.",

View File

@ -0,0 +1,74 @@
"""Google API clients: `v3-rev20260901-2.0.0` is ordered inside its generation.
The fail-green case: read loosely, every `v3-rev...` is "3", and a 2019 pin
compares equal to a 2026 revision. The fail-red case: `v2-...-2.0.0` read as
newer than `v3-...-1.25.0` because 2.0.0 > 1.25.0.
"""
from __future__ import annotations
from pathlib import Path
from dep_freshness.evaluate import judge
from dep_freshness.models import Dep, Severity
from dep_freshness.registries import maven
from dep_freshness.resolve import Answer
from dep_freshness.tests.test_gradle_registry import CENTRAL, metadata
from dep_freshness.versions import exact_pin, google_api_key, newest_per_generation
V3_OLD = "v3-rev197-1.25.0"
V3_NEW = "v3-rev20260901-2.0.0"
V2 = "v2-rev20220709-2.0.0"
def test_the_key_is_generation_then_library_then_revision():
assert google_api_key(V3_NEW)[0] == "v3"
assert google_api_key(V3_NEW)[1] > google_api_key(V3_OLD)[1]
assert google_api_key("v3-rev20260901-1.9.0")[1] < google_api_key(V3_OLD)[1]
assert google_api_key("2.0.0") is None
assert google_api_key("v3-rev197") is None
def test_newest_per_generation_keeps_every_generation():
assert (
newest_per_generation([V3_OLD, V2, V3_NEW, "junk", None, ""])
== f"{V2} {V3_NEW}"
)
assert newest_per_generation(["1.0.0"]) is None
# An older revision met AFTER the newest must not displace it.
assert newest_per_generation([V3_NEW, V3_OLD]) == V3_NEW
def test_the_pin_keeps_its_generation_prefix():
assert exact_pin(V3_OLD) == V3_OLD
def test_the_registry_answers_per_generation_when_nothing_else_parses(monkeypatch):
name = "com.google.apis:google-api-services-drive"
table = {maven.metadata_url(CENTRAL, name): metadata(V3_OLD, V2, V3_NEW)}
monkeypatch.setattr(maven, "get_text", lambda url, *a, **k: table.get(url))
assert maven.latest(name) == f"{V2} {V3_NEW}"
def dep(pinned: str = V3_OLD) -> Dep:
return Dep(
ecosystem="maven",
name="com.google.apis:google-api-services-drive",
constraint=pinned,
path=Path("libs.versions.toml"),
line=1,
pinned=pinned,
)
def test_a_pin_is_judged_inside_its_own_generation_only():
finding = judge(dep(), Answer(f"{V2} {V3_NEW}"))
assert finding.severity is Severity.STALE
assert finding.latest == V3_NEW
assert judge(dep(V3_NEW), Answer(f"{V2} {V3_NEW}")) is None
def test_a_generation_the_registry_dropped_is_unknown_not_a_pass():
finding = judge(dep(), Answer(V2))
assert finding.severity is Severity.UNKNOWN
assert "v3 generation" in finding.detail

View File

@ -0,0 +1,160 @@
"""The `upstream:` predicate: an entry holds while the fork's upstream agrees.
The rot case is the one that matters: the day upstream bumps, the entry must
die loudly rather than keep excusing a pin nobody is waiting on any more.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from dep_freshness import upstream
from dep_freshness.allowlist import AllowlistError, load
from dep_freshness.models import Dep, Exception_, Finding, Severity
from dep_freshness.registries.http import Offline
from dep_freshness.report import exceptions_block, machine_lines
from dep_freshness.tests.conftest import ALLOWLIST, STALE_PUBSPEC, write
UPSTREAM_ENTRY = """\
exceptions:
- ecosystem: pub
package: http
pinned: "1.5.0"
reason: "the project we fork from is still on 1.5.0"
blocked_by: "upstream:someone/demo"
"""
def entry(**kwargs) -> Exception_:
base = {
"ecosystem": "pub",
"package": "http",
"pinned": "1.5.0",
"reason": "r",
"blocked_by": "upstream:someone/demo",
}
return Exception_(**{**base, **kwargs})
def finding(root: Path, path: str = "pubspec.yaml") -> Finding:
dep = Dep(
ecosystem="pub",
name="http",
constraint="1.5.0",
path=root / path,
line=5,
pinned="1.5.0",
)
return Finding(dep, Severity.STALE, "1.6.0")
@pytest.fixture
def raw(monkeypatch):
"""Canned bodies per raw.githubusercontent URL; anything else is a 404."""
table: dict[str, str] = {}
monkeypatch.setattr(upstream, "get_text", lambda url, *a, **k: table.get(url))
return table
URL = "https://raw.githubusercontent.com/someone/demo/HEAD/pubspec.yaml"
def test_the_upstream_manifest_is_fetched_at_the_same_relative_path(raw, tmp_path):
raw[URL] = STALE_PUBSPEC
assert upstream.upstream_pin(entry(), finding(tmp_path), tmp_path) == "1.5.0"
assert upstream.still_pins(entry(), finding(tmp_path), tmp_path)
def test_upstream_having_moved_on_clears_the_entry(raw, tmp_path):
raw[URL] = STALE_PUBSPEC.replace("1.5.0", "1.6.0")
assert not upstream.still_pins(entry(), finding(tmp_path), tmp_path)
def test_no_upstream_manifest_or_no_such_package_is_no_pin(raw, tmp_path):
assert upstream.upstream_pin(entry(), finding(tmp_path), tmp_path) is None
raw[URL] = "name: demo\ndependencies:\n other: 1.0.0\n"
assert upstream.upstream_pin(entry(), finding(tmp_path), tmp_path) is None
def test_a_manifest_outside_the_repo_is_no_pin(raw, tmp_path):
outside = finding(tmp_path.parent, "elsewhere/pubspec.yaml")
assert upstream.upstream_pin(entry(), outside, tmp_path / "repo") is None
def test_offline_propagates_from_the_transport(monkeypatch, tmp_path):
def down(*_a, **_k):
raise Offline("down")
monkeypatch.setattr(upstream, "get_text", down)
with pytest.raises(Offline):
upstream.still_pins(entry(), finding(tmp_path), tmp_path)
def test_an_upstream_entry_parses_and_may_not_carry_an_expiry(tmp_path):
write(tmp_path, ALLOWLIST, UPSTREAM_ENTRY)
parsed = load(tmp_path)[0]
assert parsed.upstream and parsed.predicate and not parsed.transitive
assert parsed.upstream_repo == "someone/demo"
assert parsed.blocker is None
write(tmp_path, ALLOWLIST, UPSTREAM_ENTRY + ' expires: "2099-01-01"\n')
with pytest.raises(AllowlistError, match="must NOT set expires"):
load(tmp_path)
def test_an_upstream_entry_must_name_owner_slash_repo(tmp_path):
write(tmp_path, ALLOWLIST, UPSTREAM_ENTRY.replace("someone/demo", "demo"))
with pytest.raises(AllowlistError, match="upstream:<owner>/<repo>"):
load(tmp_path)
def test_the_report_treats_it_as_a_predicate(capsys):
exceptions_block([entry()], {"pub:http": False})
assert "CLEARED" in capsys.readouterr().err
assert "[still blocking]" in machine_lines([entry()])[0]
def test_end_to_end_the_entry_excuses_while_upstream_agrees(repo, run, raw):
raw[URL] = STALE_PUBSPEC
write(repo, "pubspec.yaml", STALE_PUBSPEC)
write(repo, ALLOWLIST, UPSTREAM_ENTRY)
assert run("--all", "--strict") == 0
def test_end_to_end_upstream_moving_on_fails_the_pin(repo, run, raw, capsys):
raw[URL] = STALE_PUBSPEC.replace("1.5.0", "1.6.0")
write(repo, "pubspec.yaml", STALE_PUBSPEC)
write(repo, ALLOWLIST, UPSTREAM_ENTRY)
assert run("--all", "--strict") == 1
err = capsys.readouterr().err
assert "no longer pins 1.5.0" in err
assert "CLEARED" in err
def test_end_to_end_offline_trusts_the_entry_but_degrades(
repo, run, monkeypatch, capsys
):
def down(*_a, **_k):
raise Offline("down")
monkeypatch.setattr(upstream, "get_text", down)
write(repo, "pubspec.yaml", STALE_PUBSPEC)
write(repo, ALLOWLIST, UPSTREAM_ENTRY)
assert run("--all") == 0
assert "upstream:someone/demo unverified" in capsys.readouterr().err
assert run("--all", "--strict") == 3
def test_a_transitive_entry_has_no_upstream_repo():
assert entry(blocked_by="transitive:x@1.0.0").upstream_repo is None
def test_offline_without_a_degraded_list_still_trusts_the_entry(monkeypatch, tmp_path):
from dep_freshness.check import _upstream_holds
def down(*_a, **_k):
raise Offline("down")
monkeypatch.setattr(upstream, "get_text", down)
assert _upstream_holds(entry(), finding(tmp_path), tmp_path, None)

48
dep_freshness/upstream.py Normal file
View File

@ -0,0 +1,48 @@
"""The `upstream:` predicate: does the fork's upstream still pin this version?
A fork cannot always take a release before the project it tracks does --
moko-resources 0.27.0 rewrites every string accessor across 300 files of
TachiyomiSY and would break each future upstream commit that adds a string.
"Wait for upstream" is the honest reason, and it is checkable: fetch the
upstream repository's copy of the SAME manifest, parse it with the same
parser, and compare the pin. Equal means the entry holds; different means
upstream moved and the entry is dead, which exits 2 like every other rotten
allowlist entry.
"""
from __future__ import annotations
import tempfile
from pathlib import Path
from dep_freshness._tables import GITHUB_RAW
from dep_freshness.discover import parse_manifest
from dep_freshness.models import Exception_, Finding
from dep_freshness.registries.http import get_text
def upstream_pin(entry: Exception_, finding: Finding, root: Path) -> str | None:
"""What `entry.upstream_repo` pins for this package in the same manifest.
None when upstream has no such manifest or does not declare the package;
`Offline` propagates from the transport so the caller can degrade.
"""
try:
relative = finding.dep.path.resolve().relative_to(root.resolve())
except ValueError:
return None
body = get_text(GITHUB_RAW.format(repo=entry.upstream_repo, path=relative.as_posix()))
if body is None:
return None
with tempfile.TemporaryDirectory() as scratch:
copy = Path(scratch) / finding.dep.path.name
copy.write_text(body, encoding="utf-8")
for dep in parse_manifest(copy):
if dep.ecosystem == finding.dep.ecosystem and dep.name == finding.dep.name:
return dep.pinned or dep.constraint
return None
def still_pins(entry: Exception_, finding: Finding, root: Path) -> bool:
"""True while upstream pins exactly the version the entry excuses."""
return upstream_pin(entry, finding, root) == entry.pinned

View File

@ -73,6 +73,8 @@ def is_prerelease(raw: str) -> bool:
def _canonical(raw: str) -> str:
if GOOGLE_API.match(raw.strip()):
return raw.strip() # the leading `v` is the API generation, not noise
prefix, tail = split_tag_prefix(raw)
return prefix + tail.lstrip("v")
@ -112,6 +114,35 @@ def reference(candidates) -> str | None:
return newest_stable(candidates) or _newest(candidates, prereleases=True)
# Google's API clients: `v3-rev20260901-2.0.0` is generation v3 of the API,
# revision 20260901 of the discovery document, built on library 2.0.0. No
# ordering across generations exists (v2 and v3 are different APIs), so the
# registry answers with the newest revision of EVERY generation and the
# judge picks the pin's own.
GOOGLE_API = re.compile(r"^(?P<gen>v\d+)-rev(?P<rev>\d+)-(?P<lib>\d+(?:\.\d+)*)$")
def google_api_key(raw: str) -> tuple[str, tuple[Version, int]] | None:
"""`("v3", (Version("2.0.0"), 20260901))` for a Google API client version."""
match = GOOGLE_API.match(raw.strip())
if not match:
return None
return (match.group("gen"), (Version(match.group("lib")), int(match.group("rev"))))
def newest_per_generation(candidates) -> str | None:
"""Space-joined newest Google API client per generation, or None."""
best: dict[str, tuple[tuple[Version, int], str]] = {}
for raw in candidates:
key = google_api_key(raw or "")
if key is None:
continue
gen, rank = key
if gen not in best or rank > best[gen][0]:
best[gen] = (rank, raw.strip())
return " ".join(best[gen][1] for gen in sorted(best)) or None
def behind(pinned: str, latest: str) -> bool:
"""True when `pinned` is strictly older than `latest`.