dep_freshness: gitcommit ecosystem, no-stable fallback, JitPack repo; report unknowns beside stale

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Krzysztof Rudnicki 2026-09-12 20:21:34 +02:00
parent 07e191be7b
commit 930888bc15
20 changed files with 603 additions and 94 deletions

View File

@ -18,13 +18,27 @@ CARGO: Final = "cargo"
GOMOD: Final = "gomod" GOMOD: Final = "gomod"
GITTAG: Final = "gittag" GITTAG: Final = "gittag"
MAVEN: Final = "maven" MAVEN: Final = "maven"
# A JitPack artifact pinned to a commit hash instead of a tag. "Newest" is
# the source repository's default-branch HEAD: a hash pin has no release to
# wait for, so the only reading of "current" it admits is "at HEAD".
GITCOMMIT: Final = "gitcommit"
TOOLCHAIN: Final = "toolchain" TOOLCHAIN: Final = "toolchain"
# The constraint a workflow parser writes for a version MATRIX. Standing # The constraint a workflow parser writes for a version MATRIX. Standing
# decision (kuhy, 2026-08-28): one toolchain version per repo, always newest, # decision (kuhy, 2026-08-28): one toolchain version per repo, always newest,
# so a matrix is a finding to delete rather than a range to satisfy. # so a matrix is a finding to delete rather than a range to satisfy.
MATRIX: Final = "matrix" MATRIX: Final = "matrix"
ECOSYSTEMS: Final = (PUB, PYPI, NPM, CARGO, GOMOD, GITTAG, MAVEN, TOOLCHAIN) ECOSYSTEMS: Final = (
PUB,
PYPI,
NPM,
CARGO,
GOMOD,
GITTAG,
MAVEN,
GITCOMMIT,
TOOLCHAIN,
)
# --- Registry endpoints ----------------------------------------------------- # --- Registry endpoints -----------------------------------------------------
@ -34,19 +48,25 @@ NPM_API: Final = "https://registry.npmjs.org/{name}"
CRATES_API: Final = "https://crates.io/api/v1/crates/{name}" CRATES_API: Final = "https://crates.io/api/v1/crates/{name}"
GOPROXY_API: Final = "https://proxy.golang.org/{name}/@latest" GOPROXY_API: Final = "https://proxy.golang.org/{name}/@latest"
FLUTTER_RELEASES: Final = ( FLUTTER_RELEASES: Final = (
"https://storage.googleapis.com/flutter_infra_release/releases/" "https://storage.googleapis.com/flutter_infra_release/releases/releases_linux.json"
"releases_linux.json"
) )
NODE_RELEASES: Final = "https://nodejs.org/dist/index.json" NODE_RELEASES: Final = "https://nodejs.org/dist/index.json"
GRADLE_RELEASES: Final = "https://services.gradle.org/versions/current" GRADLE_RELEASES: Final = "https://services.gradle.org/versions/current"
# Asked in order; a 404 moves on. Google first because AndroidX and the # Asked in order; a 404 moves on. Google first because AndroidX and the
# Android Gradle plugin exist nowhere else, and Central answers 404 fast. # Android Gradle plugin exist nowhere else, and Central answers 404 fast.
# JitPack last: it serves tag metadata for `com.github.*` artifacts that
# were never published anywhere else (PhotoView, DirectionalViewPager, ...).
MAVEN_REPOS: Final = ( MAVEN_REPOS: Final = (
"https://dl.google.com/dl/android/maven2/", "https://dl.google.com/dl/android/maven2/",
"https://repo1.maven.org/maven2/", "https://repo1.maven.org/maven2/",
"https://plugins.gradle.org/m2/", "https://plugins.gradle.org/m2/",
"https://jitpack.io/",
) )
UTILS_TAG_REMOTE: Final = "https://github.com/kuhyx/utils" UTILS_TAG_REMOTE: Final = "https://github.com/kuhyx/utils"
# JitPack coordinates: `com.github.<owner>:<repo>` or
# `com.github.<owner>.<repo>:<module>`; the GitHub repo is what "HEAD" means.
JITPACK_GROUP_PREFIX: Final = "com.github."
GITHUB_REMOTE: Final = "https://github.com/{repo}"
# pnpm 11 quarantines packages younger than this before it will install them, # pnpm 11 quarantines packages younger than this before it will install them,
# to blunt the window in which a compromised publish is live. The gate matches # to blunt the window in which a compromised publish is live. The gate matches
@ -67,7 +87,7 @@ CACHE_PATH_ENV: Final = "DEP_FRESHNESS_CACHE"
DEFAULT_CACHE_DIR: Final = "~/.cache/dep-freshness" DEFAULT_CACHE_DIR: Final = "~/.cache/dep-freshness"
CACHE_FILE: Final = "registry.json" CACHE_FILE: Final = "registry.json"
TTL_SECONDS: Final = 6 * 3600 TTL_SECONDS: Final = 6 * 3600
TTL_GITTAG_SECONDS: Final = 24 * 3600 TTL_GITTAG_SECONDS: Final = 24 * 3600 # also gitcommit: both cost a subprocess
HTTP_TIMEOUT: Final = 10.0 HTTP_TIMEOUT: Final = 10.0
PROBE_TIMEOUT: Final = 2.0 PROBE_TIMEOUT: Final = 2.0
MAX_WORKERS: Final = 8 MAX_WORKERS: Final = 8
@ -101,28 +121,66 @@ REQUIREMENTS_PATTERN: Final = r"^requirements.*\.txt$"
# repo being checked. Without it `--all` walks utils' own manifests and fails # repo being checked. Without it `--all` walks utils' own manifests and fails
# the consuming repo for staleness in a directory that repo does not own -- # the consuming repo for staleness in a directory that repo does not own --
# which is exactly how punchme's first green local run turned red in CI. # which is exactly how punchme's first green local run turned red in CI.
EXCLUDED_DIRS: Final = frozenset({ EXCLUDED_DIRS: Final = frozenset(
".git", "node_modules", "build", "dist", ".dart_tool", ".venv", "venv", {
"__pycache__", "target", "vendor", ".gradle", "ios", "macos", "windows", ".git",
".mypy_cache", ".pytest_cache", ".ruff_cache", "coverage", "htmlcov", "node_modules",
".idea", ".vscode", "Pods", ".fvm", ".utils", "build",
}) "dist",
".dart_tool",
".venv",
"venv",
"__pycache__",
"target",
"vendor",
".gradle",
"ios",
"macos",
"windows",
".mypy_cache",
".pytest_cache",
".ruff_cache",
"coverage",
"htmlcov",
".idea",
".vscode",
"Pods",
".fvm",
".utils",
}
)
# --- Constraint policy ------------------------------------------------------ # --- Constraint policy ------------------------------------------------------
# Q13: these resolve from the Flutter/Dart SDK or are lint packages coupled to # Q13: these resolve from the Flutter/Dart SDK or are lint packages coupled to
# it, so a caret range is allowed. An exact pin is still checked for staleness. # it, so a caret range is allowed. An exact pin is still checked for staleness.
PUB_CARET_ALLOWED: Final = frozenset({ PUB_CARET_ALLOWED: Final = frozenset(
"flutter", "flutter_test", "flutter_localizations", "flutter_driver", {
"flutter_web_plugins", "integration_test", "sky_engine", "flutter",
"flutter_lints", "very_good_analysis", "lints", "flutter_test",
}) "flutter_localizations",
"flutter_driver",
"flutter_web_plugins",
"integration_test",
"sky_engine",
"flutter_lints",
"very_good_analysis",
"lints",
}
)
# Packages that ship with the Flutter SDK: no registry version exists. # Packages that ship with the Flutter SDK: no registry version exists.
PUB_SDK_PACKAGES: Final = frozenset({ PUB_SDK_PACKAGES: Final = frozenset(
"flutter", "flutter_test", "flutter_localizations", "flutter_driver", {
"flutter_web_plugins", "integration_test", "sky_engine", "flutter",
}) "flutter_test",
"flutter_localizations",
"flutter_driver",
"flutter_web_plugins",
"integration_test",
"sky_engine",
}
)
# `dependency_overrides` is an unpinned dependency wearing a disguise. # `dependency_overrides` is an unpinned dependency wearing a disguise.
PUB_OVERRIDE_KEY: Final = "dependency_overrides" PUB_OVERRIDE_KEY: Final = "dependency_overrides"

View File

@ -8,17 +8,18 @@ user, because `--no-verify` is banned.
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass
import json import json
import os import os
from pathlib import Path
import tempfile import tempfile
import time import time
from dataclasses import dataclass
from pathlib import Path
from dep_freshness._tables import ( from dep_freshness._tables import (
CACHE_FILE, CACHE_FILE,
CACHE_PATH_ENV, CACHE_PATH_ENV,
DEFAULT_CACHE_DIR, DEFAULT_CACHE_DIR,
GITCOMMIT,
GITTAG, GITTAG,
TTL_GITTAG_SECONDS, TTL_GITTAG_SECONDS,
TTL_SECONDS, TTL_SECONDS,
@ -32,8 +33,8 @@ def cache_dir() -> Path:
def ttl_for(ecosystem: str) -> int: def ttl_for(ecosystem: str) -> int:
"""Git tags move rarely and cost a subprocess, so they cache longer.""" """Git refs move rarely and cost a subprocess, so they cache longer."""
return TTL_GITTAG_SECONDS if ecosystem == GITTAG else TTL_SECONDS return TTL_GITTAG_SECONDS if ecosystem in (GITTAG, GITCOMMIT) else TTL_SECONDS
@dataclass @dataclass

View File

@ -15,9 +15,9 @@ banned here, so a hook that hard-fails offline would leave no way to commit.
from __future__ import annotations from __future__ import annotations
import argparse import argparse
from pathlib import Path
import subprocess import subprocess
import sys import sys
from pathlib import Path
from dep_freshness import report from dep_freshness import report
from dep_freshness._tables import NPM from dep_freshness._tables import NPM
@ -213,10 +213,13 @@ def main(argv: list[str] | None = None) -> int:
print(report.as_json(failing, entries, code)) print(report.as_json(failing, entries, code))
return code return code
# Both sections print when both apply: an undeterminable package hidden
# behind a stale one only surfaces after the bump, as a fresh red run.
if unknown or resolver.degraded:
report.degraded(resolver.degraded or [f.label for f in unknown])
if real: if real:
report.violations(real, root) report.violations(real, root)
return 1 return 1
if unknown or resolver.degraded: if unknown or resolver.degraded:
report.degraded(resolver.degraded or [f.label for f in unknown])
return 3 if args.strict else 0 return 3 if args.strict else 0
return 0 return 0

View File

@ -15,12 +15,31 @@ purpose:
from __future__ import annotations from __future__ import annotations
from dep_freshness._tables import MATRIX, TOOLCHAIN from dep_freshness._tables import GITCOMMIT, MATRIX, TOOLCHAIN
from dep_freshness.constraints import lower_bound, satisfies from dep_freshness.constraints import lower_bound, satisfies
from dep_freshness.models import Dep, Finding, Severity from dep_freshness.models import Dep, Finding, Severity
from dep_freshness.resolve import Answer from dep_freshness.resolve import Answer
from dep_freshness.versions import behind from dep_freshness.versions import behind
# What JitPack accepts as a version, and what a bump pastes into the catalog.
_SHORT_SHA = 10
def _commit(dep: Dep, head: str) -> Finding | None:
"""A hash pin is current when the default branch's HEAD still starts with it."""
short = head[:_SHORT_SHA]
if not dep.pinned:
return Finding(dep, Severity.UNPINNED, short, detail=f"pin a commit ({short})")
if head.lower().startswith(dep.pinned.lower()):
return None
return Finding(
dep,
Severity.STALE,
short,
detail="the source repository's default branch has moved on; a commit "
"pin has no release to wait for",
)
def _toolchain(dep: Dep, latest: str) -> Finding | None: def _toolchain(dep: Dep, latest: str) -> Finding | None:
if dep.pinned: if dep.pinned:
@ -86,6 +105,8 @@ def judge(dep: Dep, answer: Answer) -> Finding | None:
if dep.ecosystem == TOOLCHAIN: if dep.ecosystem == TOOLCHAIN:
return _toolchain(dep, latest) return _toolchain(dep, latest)
if dep.ecosystem == GITCOMMIT:
return _commit(dep, latest)
if dep.peer: if dep.peer:
# A peerDependency declares what a CONSUMER may bring, so exact-pinning # A peerDependency declares what a CONSUMER may bring, so exact-pinning

View File

@ -33,7 +33,7 @@ from typing import Any
import tomllib import tomllib
from dep_freshness._tables import MAVEN, TOOLCHAIN from dep_freshness._tables import GITCOMMIT, JITPACK_GROUP_PREFIX, MAVEN, TOOLCHAIN
from dep_freshness.models import Dep from dep_freshness.models import Dep
from dep_freshness.parsers._lines import index from dep_freshness.parsers._lines import index
from dep_freshness.versions import exact_pin from dep_freshness.versions import exact_pin
@ -43,6 +43,15 @@ WRAPPER_NAME = "gradle-wrapper.properties"
_DISTRIBUTION = re.compile( _DISTRIBUTION = re.compile(
r"gradle-(?P<version>\d+(?:\.\d+)*(?:-[0-9A-Za-z.-]+?)?)-(?:bin|all)\.zip" r"gradle-(?P<version>\d+(?:\.\d+)*(?:-[0-9A-Za-z.-]+?)?)-(?:bin|all)\.zip"
) )
# JitPack accepts any commit as a version; 7 to 40 hex digits and nothing
# else is one. `1.0.0` never matches (dots), and a tag like `v2.3.0` has a
# letter outside a-f.
_COMMIT = re.compile(r"^[0-9a-f]{7,40}$")
def _commit_pin(name: str, version: str) -> bool:
"""True for `com.github.*` at a commit hash: a gitcommit dep, not maven."""
return name.startswith(JITPACK_GROUP_PREFIX) and bool(_COMMIT.match(version))
def _rich(value: Any) -> str: def _rich(value: Any) -> str:
@ -104,14 +113,17 @@ def parse_catalog(path: Path) -> list[Dep]:
continue # BOM-managed, or a dangling ref the build itself rejects continue # BOM-managed, or a dangling ref the build itself rejects
if plugin: if plugin:
name = f"{name}:{name}.gradle.plugin" name = f"{name}:{name}.gradle.plugin"
ecosystem, pinned = MAVEN, exact_pin(version)
if _commit_pin(name, version):
ecosystem, pinned = GITCOMMIT, version.strip()
deps.append( deps.append(
Dep( Dep(
ecosystem=MAVEN, ecosystem=ecosystem,
name=name, name=name,
constraint=version, constraint=version,
path=path, path=path,
line=line, line=line,
pinned=exact_pin(version), pinned=pinned,
) )
) )
return deps return deps

View File

@ -0,0 +1,38 @@
"""`git ls-remote` as the one subprocess the registries share.
Deliberately not the GitHub REST API: that allows 60 unauthenticated requests
an hour, which ~20 consuming manifests would exhaust on a single `--all` run.
`ls-remote` is unmetered and answers tags and HEAD alike.
"""
from __future__ import annotations
import subprocess
from dep_freshness.registries.http import Offline, host_reachable
def ls_remote(remote: str, *args: str) -> list[str]:
"""Every `<sha>\\t<ref>` line `git ls-remote` prints for `remote`.
`--options` go before the URL and ref patterns (`HEAD`) after it: git
reads the first positional as the repository, so `ls-remote HEAD <url>`
fails with "'HEAD' does not appear to be a git repository".
"""
if not host_reachable(remote):
raise Offline(remote)
options = [a for a in args if a.startswith("-")]
patterns = [a for a in args if not a.startswith("-")]
try:
result = subprocess.run(
["git", "ls-remote", *options, remote, *patterns],
capture_output=True,
text=True,
timeout=20,
check=False,
)
except (OSError, subprocess.SubprocessError) as exc:
raise Offline(f"{remote}: {exc}") from exc
if result.returncode != 0:
raise Offline(f"{remote}: git ls-remote exit {result.returncode}")
return [line.strip() for line in result.stdout.splitlines() if line.strip()]

View File

@ -8,7 +8,7 @@ from __future__ import annotations
from dep_freshness._tables import CRATES_API from dep_freshness._tables import CRATES_API
from dep_freshness.registries.http import get_json from dep_freshness.registries.http import get_json
from dep_freshness.versions import newest_stable from dep_freshness.versions import reference
def latest(name: str) -> str | None: def latest(name: str) -> str | None:
@ -19,7 +19,5 @@ def latest(name: str) -> str | None:
stable = crate.get("max_stable_version") stable = crate.get("max_stable_version")
if stable: if stable:
return str(stable) return str(stable)
live = [ live = [v.get("num") for v in payload.get("versions") or [] if not v.get("yanked")]
v.get("num") for v in payload.get("versions") or [] if not v.get("yanked") return reference(live)
]
return newest_stable(live)

View File

@ -0,0 +1,47 @@
"""JitPack artifacts pinned to a commit: "newest" is the default branch's HEAD.
Seven of TachiyomiSY's dependencies are `com.github.<owner>:<repo>` at a
short hash. No registry lists a "latest" for those, and a hash is not a
version, so the gate's question becomes the only one a commit pin admits:
is the pinned commit still what the source repository's default branch
points at? (kuhy, 2026-09-12: chosen over never-expiring allowlist entries.)
The answer is the full HEAD sha; `evaluate` compares by prefix so a 7-, 10-
or 40-character pin all work, and reports the first ten characters, which
is what JitPack accepts as a version.
"""
from __future__ import annotations
from dep_freshness._tables import GITHUB_REMOTE, JITPACK_GROUP_PREFIX
from dep_freshness.registries._git import ls_remote
def source_repo(name: str) -> str | None:
"""`owner/repo` behind a JitPack coordinate, or None for anything else.
`com.github.mihonapp:image-decoder` -> `mihonapp/image-decoder`;
`com.github.arkon.FlexibleAdapter:flexible-adapter` -> `arkon/FlexibleAdapter`
(a multi-module build names the repo in the group, the module after the
colon).
"""
group, sep, artifact = name.partition(":")
if not sep or not artifact or not group.startswith(JITPACK_GROUP_PREFIX):
return None
parts = group[len(JITPACK_GROUP_PREFIX) :].split(".")
if not parts[0]:
return None
repo = parts[1] if len(parts) > 1 and parts[1] else artifact
return f"{parts[0]}/{repo}"
def latest(name: str) -> str | None:
"""The default branch's HEAD sha, or None when `name` is not a JitPack pin."""
repo = source_repo(name)
if repo is None:
return None
for line in ls_remote(GITHUB_REMOTE.format(repo=repo), "HEAD"):
sha, _, ref = line.partition("\t")
if ref.strip() == "HEAD" and sha:
return sha.strip()
return None

View File

@ -1,18 +1,15 @@
"""Shared libraries consumed by git tag out of `kuhyx/utils`. """Shared libraries consumed by git tag out of `kuhyx/utils`.
"Newest" here is the newest tag matching `<package>-vX.Y.Z`, not a registry "Newest" here is the newest tag matching `<package>-vX.Y.Z`, not a registry
answer. Deliberately `git ls-remote` rather than the GitHub REST API: that API answer. The transport is `git ls-remote` (see `_git.py`), never the REST API.
allows 60 unauthenticated requests an hour, which ~20 consuming manifests would
exhaust on a single `--all` run.
""" """
from __future__ import annotations from __future__ import annotations
import re import re
import subprocess
from dep_freshness._tables import UTILS_TAG_REMOTE from dep_freshness._tables import UTILS_TAG_REMOTE
from dep_freshness.registries.http import Offline, host_reachable from dep_freshness.registries._git import ls_remote
from dep_freshness.versions import newest_stable from dep_freshness.versions import newest_stable
_TAG = re.compile(r"refs/tags/(?P<name>[A-Za-z0-9_.-]+)-v(?P<version>[0-9][^\s]*)$") _TAG = re.compile(r"refs/tags/(?P<name>[A-Za-z0-9_.-]+)-v(?P<version>[0-9][^\s]*)$")
@ -20,23 +17,9 @@ _TAG = re.compile(r"refs/tags/(?P<name>[A-Za-z0-9_.-]+)-v(?P<version>[0-9][^\s]*
def latest(package: str, remote: str = UTILS_TAG_REMOTE) -> str | None: def latest(package: str, remote: str = UTILS_TAG_REMOTE) -> str | None:
"""Highest semver tag prefixed with `<package>-v`, or None if there is none.""" """Highest semver tag prefixed with `<package>-v`, or None if there is none."""
if not host_reachable(remote):
raise Offline(remote)
try:
result = subprocess.run(
["git", "ls-remote", "--tags", "--refs", remote],
capture_output=True,
text=True,
timeout=20,
check=False,
)
except (OSError, subprocess.SubprocessError) as exc:
raise Offline(f"{remote}: {exc}") from exc
if result.returncode != 0:
raise Offline(f"{remote}: git ls-remote exit {result.returncode}")
found = [] found = []
for line in result.stdout.splitlines(): for line in ls_remote(remote, "--tags", "--refs"):
match = _TAG.search(line.strip()) match = _TAG.search(line)
if match and match.group("name") == package: if match and match.group("name") == package:
found.append(match.group("version")) found.append(match.group("version"))
return newest_stable(found) return newest_stable(found)

View File

@ -1,14 +1,19 @@
"""Maven repositories: newest stable of `group:artifact`, from maven-metadata. """Maven repositories: newest stable of `group:artifact`, from maven-metadata.
Three repositories, asked in order until one knows the artifact: Google Four repositories, asked in order until one knows the artifact: Google
Maven (every `androidx.*` and `com.android.*` artifact lives ONLY there), Maven (every `androidx.*` and `com.android.*` artifact lives ONLY there),
Maven Central, and the Gradle plugin portal (plugin marker artifacts Maven Central, the Gradle plugin portal (plugin marker artifacts
`<id>:<id>.gradle.plugin`). A 404 means "not here", not "not anywhere", so `<id>:<id>.gradle.plugin`) and JitPack (tag metadata for `com.github.*`
it moves on; anything else surfaces as `Offline` from the HTTP layer. artifacts published nowhere else). A 404 means "not here", not "not
anywhere", so it moves on; anything else surfaces as `Offline` from the HTTP
layer.
`<latest>` and `<release>` in the metadata are not trusted: AndroidX writes `<latest>` and `<release>` in the metadata are not trusted: AndroidX writes
`1.2.0-alpha03` into `<latest>` routinely. The full `<versions>` list goes `1.2.0-alpha03` into `<latest>` routinely. The full `<versions>` list goes
through `newest_stable` like every other ecosystem. through `newest_stable` like every other ecosystem. Only when NO repository
has a stable release at all does the newest pre-release become the reference
(`versions.reference`), which is how `androidx.biometric:biometric-ktx`,
alpha-only since 2021, gets a checkable answer.
""" """
from __future__ import annotations from __future__ import annotations
@ -17,7 +22,7 @@ from xml.etree import ElementTree
from dep_freshness._tables import MAVEN_REPOS from dep_freshness._tables import MAVEN_REPOS
from dep_freshness.registries.http import get_text from dep_freshness.registries.http import get_text
from dep_freshness.versions import newest_stable from dep_freshness.versions import newest_stable, reference
def metadata_url(repo: str, name: str) -> str | None: def metadata_url(repo: str, name: str) -> str | None:
@ -42,6 +47,7 @@ def versions_in(xml: str) -> list[str]:
def latest(name: str) -> str | None: def latest(name: str) -> str | None:
seen: list[str] = []
for repo in MAVEN_REPOS: for repo in MAVEN_REPOS:
url = metadata_url(repo, name) url = metadata_url(repo, name)
if url is None: if url is None:
@ -49,7 +55,9 @@ def latest(name: str) -> str | None:
body = get_text(url) body = get_text(url)
if body is None: if body is None:
continue continue
stable = newest_stable(versions_in(body)) versions = versions_in(body)
stable = newest_stable(versions)
if stable is not None: if stable is not None:
return stable return stable
return None seen.extend(versions)
return reference(seen)

View File

@ -9,7 +9,7 @@ from __future__ import annotations
from dep_freshness._tables import NPM_ACCEPT, NPM_API from dep_freshness._tables import NPM_ACCEPT, NPM_API
from dep_freshness.registries.http import get_json from dep_freshness.registries.http import get_json
from dep_freshness.versions import is_prerelease, newest_stable from dep_freshness.versions import is_prerelease, reference
def latest(name: str) -> str | None: def latest(name: str) -> str | None:
@ -20,4 +20,4 @@ def latest(name: str) -> str | None:
tag = (payload.get("dist-tags") or {}).get("latest") tag = (payload.get("dist-tags") or {}).get("latest")
if tag and not is_prerelease(str(tag)): if tag and not is_prerelease(str(tag)):
return str(tag) return str(tag)
return newest_stable((payload.get("versions") or {}).keys()) return reference((payload.get("versions") or {}).keys())

View File

@ -4,7 +4,7 @@ from __future__ import annotations
from dep_freshness._tables import PUB_API from dep_freshness._tables import PUB_API
from dep_freshness.registries.http import get_json from dep_freshness.registries.http import get_json
from dep_freshness.versions import newest_stable from dep_freshness.versions import newest_stable, reference
def latest(name: str) -> str | None: def latest(name: str) -> str | None:
@ -20,4 +20,4 @@ def latest(name: str) -> str | None:
if candidate and newest_stable([candidate]): if candidate and newest_stable([candidate]):
return candidate return candidate
versions = [v.get("version") for v in payload.get("versions") or []] versions = [v.get("version") for v in payload.get("versions") or []]
return newest_stable(versions) return reference(versions)

View File

@ -10,7 +10,7 @@ from __future__ import annotations
from dep_freshness._tables import PYPI_ACCEPT, PYPI_API from dep_freshness._tables import PYPI_ACCEPT, PYPI_API
from dep_freshness.registries.http import get_json from dep_freshness.registries.http import get_json
from dep_freshness.versions import newest_stable from dep_freshness.versions import reference
_SUFFIXES = (".tar.gz", ".zip", ".whl", ".tar.bz2", ".egg") _SUFFIXES = (".tar.gz", ".zip", ".whl", ".tar.bz2", ".egg")
@ -44,5 +44,5 @@ def latest(name: str) -> str | None:
if found: if found:
live.add(found) live.add(found)
if live: if live:
return newest_stable(live) return reference(live)
return newest_stable(versions) return reference(versions)

View File

@ -12,6 +12,7 @@ from dataclasses import dataclass
from dep_freshness._tables import ( from dep_freshness._tables import (
CARGO, CARGO,
GITCOMMIT,
GITTAG, GITTAG,
GOMOD, GOMOD,
MAVEN, MAVEN,
@ -23,7 +24,16 @@ from dep_freshness._tables import (
) )
from dep_freshness.cache import Cache from dep_freshness.cache import Cache
from dep_freshness.models import Dep from dep_freshness.models import Dep
from dep_freshness.registries import cargo, gittag, gomod, maven, npmjs, pub, pypi from dep_freshness.registries import (
cargo,
gitcommit,
gittag,
gomod,
maven,
npmjs,
pub,
pypi,
)
from dep_freshness.registries import toolchain as tc from dep_freshness.registries import toolchain as tc
from dep_freshness.registries.http import Offline from dep_freshness.registries.http import Offline
@ -35,6 +45,7 @@ _LOOKUP = {
GOMOD: gomod.latest, GOMOD: gomod.latest,
GITTAG: gittag.latest, GITTAG: gittag.latest,
MAVEN: maven.latest, MAVEN: maven.latest,
GITCOMMIT: gitcommit.latest,
} }
@ -43,8 +54,8 @@ class Answer:
"""Latest stable for one package, and how confident we are in it.""" """Latest stable for one package, and how confident we are in it."""
version: str | None version: str | None
stale_cache_days: float | None = None # served from an expired entry stale_cache_days: float | None = None # served from an expired entry
unavailable: bool = False # no network and no cache at all unavailable: bool = False # no network and no cache at all
def _toolchain_latest(name: str) -> str | None: def _toolchain_latest(name: str) -> str | None:

View File

@ -111,3 +111,23 @@ def _offline(_ecosystem, _name):
def test_the_cache_directory_is_honoured(cache_dir): def test_the_cache_directory_is_honoured(cache_dir):
assert os.environ["DEP_FRESHNESS_CACHE"] == str(cache_dir) assert os.environ["DEP_FRESHNESS_CACHE"] == str(cache_dir)
def test_an_undeterminable_package_is_reported_beside_a_stale_one(
repo, run, capsys, monkeypatch
):
"""Hidden behind the stale finding it only surfaced after the bump, as a
fresh red run nobody could have planned for."""
from dep_freshness.registries.http import Offline
def fetch(ecosystem, package):
if package == "http":
return "1.7.0"
raise Offline(package)
monkeypatch.setattr("dep_freshness.resolve._fetch", fetch)
write(repo, "pubspec.yaml", STALE_PUBSPEC)
assert run("--all", "--strict") == 1
err = capsys.readouterr().err
assert "pub:http" in err
assert "DEGRADED" in err

View File

@ -0,0 +1,175 @@
"""JitPack commit pins: "newest" is the source repo's default-branch HEAD.
The fail-green case here is a hash the version parser cannot read: `behind`
returns False for unparseable input, so without its own ecosystem a moved
default branch would pass silently. Every path below is pinned down.
"""
from __future__ import annotations
import subprocess
from pathlib import Path
import pytest
from dep_freshness.cache import ttl_for
from dep_freshness.evaluate import judge
from dep_freshness.models import Dep, Severity
from dep_freshness.parsers import gradle
from dep_freshness.registries import _git, gitcommit
from dep_freshness.registries.http import Offline
from dep_freshness.resolve import _LOOKUP, Answer
from dep_freshness.tests.conftest import write
HEAD = "844a07002c3d2f2b68a6b26a25e8e401777c73e7"
@pytest.fixture
def ls_remote(monkeypatch):
monkeypatch.setattr(_git, "host_reachable", lambda _url: True)
calls: list[list[str]] = []
def install(stdout="", returncode=0, error=None):
def fake_run(args, **_kwargs):
calls.append(args)
if error:
raise error
return subprocess.CompletedProcess(args, returncode, stdout, "")
monkeypatch.setattr(subprocess, "run", fake_run)
return calls
return install
@pytest.mark.parametrize(
"name,expected",
[
("com.github.mihonapp:image-decoder", "mihonapp/image-decoder"),
("com.github.arkon.FlexibleAdapter:flexible-adapter", "arkon/FlexibleAdapter"),
("com.github.zhanghai.quickjs-java:quickjs-android", "zhanghai/quickjs-java"),
("com.github.owner.repo.deeper:module", "owner/repo"),
("com.github.owner.:module", "owner/module"),
("io.coil-kt:coil", None),
("com.github.:module", None),
("com.github.owner", None),
("com.github.owner:", None),
],
)
def test_the_github_repo_behind_a_jitpack_coordinate(name, expected):
assert gitcommit.source_repo(name) == expected
def test_latest_is_the_default_branch_head(ls_remote):
calls = ls_remote(f"{HEAD}\tHEAD\n{HEAD}\trefs/heads/master\n")
assert gitcommit.latest("com.github.arkon.FlexibleAdapter:flexible-adapter") == HEAD
assert calls[-1] == [
"git",
"ls-remote",
"https://github.com/arkon/FlexibleAdapter",
"HEAD",
]
def test_no_head_line_means_no_answer(ls_remote):
ls_remote("\n \nabc\trefs/heads/dev\n")
assert gitcommit.latest("com.github.mihonapp:image-decoder") is None
def test_a_non_jitpack_coordinate_never_touches_git(ls_remote):
calls = ls_remote(f"{HEAD}\tHEAD\n")
assert gitcommit.latest("io.coil-kt:coil") is None
assert calls == []
def test_ls_remote_failures_degrade_to_offline(ls_remote, monkeypatch):
ls_remote(returncode=128)
with pytest.raises(Offline):
gitcommit.latest("com.github.mihon:unifile")
ls_remote(error=OSError("no git"))
with pytest.raises(Offline):
gitcommit.latest("com.github.mihon:unifile")
monkeypatch.setattr(_git, "host_reachable", lambda _url: False)
with pytest.raises(Offline):
gitcommit.latest("com.github.mihon:unifile")
CATALOG = """\
[versions]
unifile = "08f224c8f9"
photoView = "2.3.0"
tagged = "abc1234"
[libraries]
unifile = { module = "com.github.mihon:unifile", version.ref = "unifile" }
photoView = { module = "com.github.chrisbanes:PhotoView", version.ref = "photoView" }
inline = "com.github.mihonapp:image-decoder:e03b81e18a"
central = { module = "io.coil-kt:coil", version = "deadbeef" }
short = { module = "com.github.o:r", version = "abc12" }
tagged = { module = "org.example:lib", version.ref = "tagged" }
"""
def test_the_catalog_parser_tells_commit_pins_from_versions(tmp_path):
deps = {
d.name: d
for d in gradle.parse_catalog(write(tmp_path, "libs.versions.toml", CATALOG))
}
assert (
deps["com.github.mihon:unifile"].ecosystem,
deps["com.github.mihon:unifile"].pinned,
) == (
"gitcommit",
"08f224c8f9",
)
assert deps["com.github.mihonapp:image-decoder"].ecosystem == "gitcommit"
assert deps["com.github.chrisbanes:PhotoView"].ecosystem == "maven"
assert deps["com.github.chrisbanes:PhotoView"].pinned == "2.3.0"
# A hash-shaped version outside com.github.* is still a Maven version
# (an unpinned one, since it is not exact), and six hex digits is too few.
assert deps["io.coil-kt:coil"].ecosystem == "maven"
assert deps["io.coil-kt:coil"].pinned is None
assert deps["com.github.o:r"].ecosystem == "maven"
assert deps["org.example:lib"].ecosystem == "maven"
def dep(pinned: str | None = "844a07002c") -> Dep:
return Dep(
ecosystem="gitcommit",
name="com.github.arkon.FlexibleAdapter:flexible-adapter",
constraint=pinned or "",
path=Path("libs.versions.toml"),
line=1,
pinned=pinned,
)
def test_a_pin_at_head_passes_whatever_its_length_or_case():
assert judge(dep("844a07002c"), Answer(HEAD)) is None
assert judge(dep("844A070"), Answer(HEAD)) is None
assert judge(dep(HEAD), Answer(HEAD)) is None
def test_a_pin_behind_head_is_stale_and_reports_the_short_sha():
finding = judge(dep("c8013533"), Answer(HEAD))
assert finding.severity is Severity.STALE
assert finding.latest == "844a07002c"
assert "default branch has moved" in finding.detail
def test_a_missing_pin_is_reported_unpinned_with_head_to_use():
finding = judge(dep(None), Answer(HEAD))
assert finding.severity is Severity.UNPINNED
assert "844a07002c" in finding.detail
def test_no_head_at_all_is_unknown_not_a_pass():
assert judge(dep(), Answer(None)).severity is Severity.UNKNOWN
def test_commit_answers_cache_as_long_as_tags():
assert ttl_for("gitcommit") == ttl_for("gittag") > ttl_for("maven")
def test_the_resolver_routes_gitcommit_to_the_git_registry():
assert _LOOKUP["gitcommit"] is gitcommit.latest

View File

@ -1,4 +1,4 @@
"""Maven metadata across three repositories, and the Gradle release feed.""" """Maven metadata across four repositories, and the Gradle release feed."""
from __future__ import annotations from __future__ import annotations
@ -13,7 +13,7 @@ from dep_freshness.registries.http import Offline
from dep_freshness.resolve import _toolchain_latest from dep_freshness.resolve import _toolchain_latest
from dep_freshness.tests.conftest import write from dep_freshness.tests.conftest import write
GOOGLE, CENTRAL, PORTAL = maven.MAVEN_REPOS GOOGLE, CENTRAL, PORTAL, JITPACK = maven.MAVEN_REPOS
def metadata(*versions: str, latest: str | None = None) -> str: def metadata(*versions: str, latest: str | None = None) -> str:
@ -68,6 +68,31 @@ def test_a_repo_with_only_prereleases_does_not_stop_the_search(repos):
assert maven.latest("g:a") == "0.9.0" assert maven.latest("g:a") == "0.9.0"
def test_jitpack_is_asked_last_and_reads_tag_metadata(repos):
"""PhotoView was never published anywhere but JitPack; `v`-tags and bare
tags both appear and the newest wins."""
name = "com.github.chrisbanes:PhotoView"
repos[maven.metadata_url(JITPACK, name)] = metadata("v1.3.1", "2.2.0", "2.3.0")
assert maven.latest(name) == "2.3.0"
def test_a_project_named_tag_is_read_as_its_version(repos):
"""`java-nat-sort` is tagged `natural-comparator-1.1`, not `1.1`."""
name = "com.github.gpanther:java-nat-sort"
repos[maven.metadata_url(JITPACK, name)] = metadata("natural-comparator-1.1")
assert maven.latest(name) == "natural-comparator-1.1"
def test_a_package_with_no_stable_anywhere_falls_back_to_newest_prerelease(repos):
"""biometric-ktx has been alpha-only since 2021: the newest alpha is the
reference, and a stable release in ANY repo would still win over it."""
name = "androidx.biometric:biometric-ktx"
repos[maven.metadata_url(GOOGLE, name)] = metadata(
"1.2.0-alpha05", "1.4.0-alpha02", "1.3.0-alpha01"
)
assert maven.latest(name) == "1.4.0-alpha02"
def test_unknown_everywhere_or_malformed_name_is_none(repos): def test_unknown_everywhere_or_malformed_name_is_none(repos):
assert maven.latest("g:nowhere") is None assert maven.latest("g:nowhere") is None
assert maven.latest("bare") is None assert maven.latest("bare") is None

View File

@ -6,7 +6,7 @@ import subprocess
import pytest import pytest
from dep_freshness.registries import gittag from dep_freshness.registries import _git, gittag
from dep_freshness.registries import toolchain as tc from dep_freshness.registries import toolchain as tc
from dep_freshness.registries.http import Offline from dep_freshness.registries.http import Offline
@ -75,7 +75,7 @@ LS_REMOTE = (
@pytest.fixture @pytest.fixture
def ls_remote(monkeypatch): def ls_remote(monkeypatch):
monkeypatch.setattr(gittag, "host_reachable", lambda _url: True) monkeypatch.setattr(_git, "host_reachable", lambda _url: True)
def install(stdout="", returncode=0, error=None): def install(stdout="", returncode=0, error=None):
def fake_run(*_args, **_kwargs): def fake_run(*_args, **_kwargs):
@ -114,6 +114,6 @@ def test_a_missing_git_binary_degrades_to_offline(ls_remote):
def test_an_unreachable_remote_degrades_to_offline(monkeypatch): def test_an_unreachable_remote_degrades_to_offline(monkeypatch):
monkeypatch.setattr(gittag, "host_reachable", lambda _url: False) monkeypatch.setattr(_git, "host_reachable", lambda _url: False)
with pytest.raises(Offline): with pytest.raises(Offline):
gittag.latest("crdt_sync_dart") gittag.latest("crdt_sync_dart")

View File

@ -4,7 +4,14 @@ from __future__ import annotations
import pytest import pytest
from dep_freshness.versions import behind, exact_pin, is_prerelease, newest_stable from dep_freshness.versions import (
behind,
exact_pin,
is_prerelease,
newest_stable,
reference,
split_tag_prefix,
)
@pytest.mark.parametrize("raw", [ @pytest.mark.parametrize("raw", [
@ -58,3 +65,49 @@ def test_behind_never_fires_on_unparseable_input():
"""An unpinned constraint is UNPINNED, never dressed up as staleness.""" """An unpinned constraint is UNPINNED, never dressed up as staleness."""
assert not behind("^1.6.0", "1.7.0") assert not behind("^1.6.0", "1.7.0")
assert not behind("1.6.0", "latest") assert not behind("1.6.0", "latest")
@pytest.mark.parametrize("raw,expected", [
("natural-comparator-1.1", ("natural-comparator-", "1.1")),
("release-v2.0", ("release-", "v2.0")),
("2.3.0-rc1", ("", "2.3.0-rc1")),
("v1.0.0", ("", "v1.0.0")),
("alpha", ("", "alpha")),
])
def test_a_project_named_tag_splits_into_prefix_and_version(raw, expected):
"""JitPack serves java-nat-sort only as `natural-comparator-1.1`."""
assert split_tag_prefix(raw) == expected
def test_a_project_named_tag_is_a_stable_version_with_its_prefix_kept():
assert not is_prerelease("natural-comparator-1.1")
assert is_prerelease("natural-comparator-1.2-rc1")
assert newest_stable(["natural-comparator-1.1", "natural-comparator-1.0"]) == (
"natural-comparator-1.1"
)
assert behind("natural-comparator-1.0", "natural-comparator-1.1")
assert exact_pin("natural-comparator-1.1") == "natural-comparator-1.1"
assert exact_pin("version-v1.2") == "version-1.2"
def test_reference_is_newest_stable_when_one_exists():
assert reference(["1.0.0", "2.0.0-alpha01"]) == "1.0.0"
def test_reference_falls_back_to_newest_prerelease_only_when_nothing_is_stable():
"""androidx.biometric:biometric-ktx: alpha-only since 2021."""
assert reference(["1.2.0-alpha05", "1.4.0-alpha02", "1.3.0-alpha01"]) == "1.4.0-alpha02"
assert reference(iter(["1.4.0-alpha02", "junk"])) == "1.4.0-alpha02"
assert reference([]) is None
assert reference(["junk", ""]) is None
def test_an_api_generation_is_not_a_project_name():
"""Google's `v3-rev197-1.25.0`: splitting on `v3-rev197-` compared the
Drive v3 client against a v2 artifact and called v2 newer."""
assert split_tag_prefix("v3-rev197-1.25.0") == ("", "v3-rev197-1.25.0")
assert is_prerelease("v3-rev197-1.25.0")
assert newest_stable(["v3-rev197-1.25.0", "v2-rev20220709-2.0.0"]) is None
# ... and the no-stable fallback must not pick one up either: loosely
# parsed, every v3 revision is "3" and the 2019 pin reads as current.
assert reference(["v3-rev197-1.25.0", "v3-rev20250723-2.0.0"]) is None

View File

@ -3,7 +3,8 @@
The dangerous bug class here fails *green*: npm publishes pre-releases to The dangerous bug class here fails *green*: npm publishes pre-releases to
`dist-tags.latest` and crates.io's `max_version` includes them, so an adapter `dist-tags.latest` and crates.io's `max_version` includes them, so an adapter
that trusts either reports "up to date" while the pin is behind. Every that trusts either reports "up to date" while the pin is behind. Every
"latest" that leaves this package goes through `newest_stable`. "latest" that leaves this package goes through `newest_stable` or, for a
package that has never shipped a stable release, `reference`.
""" """
from __future__ import annotations from __future__ import annotations
@ -19,15 +20,38 @@ _SEMVER = re.compile(
r"(?:-(?P<pre>[0-9A-Za-z.-]+))?" r"(?:-(?P<pre>[0-9A-Za-z.-]+))?"
r"(?:\+(?P<build>[0-9A-Za-z.-]+))?$" r"(?:\+(?P<build>[0-9A-Za-z.-]+))?$"
) )
# A tag that carries its project name: JitPack serves `java-nat-sort` only as
# `natural-comparator-1.1`. Only a digit-free name that STARTS with a letter
# and is followed by the version counts: `2.3.0-rc1` is untouched and stays a
# pre-release, and Google's `v3-rev197-1.25.0` API clients are NOT split,
# because `v3-` is an API generation, not a project name, and reading it as
# one proposed a `v2-...-2.0.0` artifact as "newer".
_TAG_PREFIX = re.compile(r"^(?P<prefix>[A-Za-z][A-Za-z_.-]*?-)(?=v?\d)")
def parse(raw: str) -> Version | None: def split_tag_prefix(raw: str) -> tuple[str, str]:
"""A comparable version, or None when the string is not one.""" """`("natural-comparator-", "1.1")`; an unprefixed version keeps `("", ...)`."""
text = raw.strip().lstrip("v") text = raw.strip()
match = _TAG_PREFIX.match(text)
if not match:
return ("", text)
return (match.group("prefix"), text[match.end() :])
def parse(raw: str, strict: bool = False) -> Version | None:
"""A comparable version, or None when the string is not one.
`strict` refuses the semver-core fallback: `v3-rev197-1.25.0` reads as
`3` loosely, which is enough to order it but says nothing about what it
is, and the pre-release fallback below must not build on that.
"""
tail = split_tag_prefix(raw)[1]
try: try:
return Version(text) return Version(tail.lstrip("v"))
except InvalidVersion: except InvalidVersion:
match = _SEMVER.match(raw.strip()) if strict:
return None
match = _SEMVER.match(tail)
if not match: if not match:
return None return None
# `core` is `\d+(\.\d+){0,2}`, which Version always accepts. # `core` is `\d+(\.\d+){0,2}`, which Version always accepts.
@ -41,26 +65,53 @@ def is_prerelease(raw: str) -> bool:
silently normalises some semver pre-release tags (`-alpha` survives, but silently normalises some semver pre-release tags (`-alpha` survives, but
an unknown suffix can be dropped) and a dropped suffix reads as stable. an unknown suffix can be dropped) and a dropped suffix reads as stable.
""" """
if "-" in raw.split("+", 1)[0].lstrip("v"): tail = split_tag_prefix(raw)[1]
if "-" in tail.split("+", 1)[0].lstrip("v"):
return True return True
version = parse(raw) version = parse(raw)
return bool(version and (version.is_prerelease or version.is_devrelease)) return bool(version and (version.is_prerelease or version.is_devrelease))
def newest_stable(candidates) -> str | None: def _canonical(raw: str) -> str:
"""The highest non-pre-release version in `candidates`, or None.""" prefix, tail = split_tag_prefix(raw)
return prefix + tail.lstrip("v")
def _newest(candidates, prereleases: bool) -> str | None:
best: tuple[Version, str] | None = None best: tuple[Version, str] | None = None
for raw in candidates: for raw in candidates:
if not raw or is_prerelease(raw): if not raw or (not prereleases and is_prerelease(raw)):
continue continue
version = parse(raw) version = parse(raw, strict=prereleases)
if version is None: if version is None:
continue continue
if best is None or version > best[0]: if best is None or version > best[0]:
best = (version, str(raw).strip().lstrip("v")) best = (version, _canonical(str(raw)))
return best[1] if best else None return best[1] if best else None
def newest_stable(candidates) -> str | None:
"""The highest non-pre-release version in `candidates`, or None."""
return _newest(candidates, prereleases=False)
def reference(candidates) -> str | None:
"""Newest stable, or newest pre-release when no stable was EVER shipped.
`androidx.biometric:biometric-ktx` has been alpha-only since 2021; holding
such a package to "newest stable" is a question with no answer, and an
allowlist entry for it would need renewing every 90 days forever. The
fallback is a predicate: the day a stable release appears, it wins.
Only pre-releases PEP 440 can read qualify. Google's Drive client ships
`v3-rev197-1.25.0`-style versions that are not pre-releases at all; the
loose parser orders them by API generation, and a fallback built on that
once declared a 2019 pin current against every newer revision.
"""
candidates = list(candidates)
return newest_stable(candidates) or _newest(candidates, prereleases=True)
def behind(pinned: str, latest: str) -> bool: def behind(pinned: str, latest: str) -> bool:
"""True when `pinned` is strictly older than `latest`. """True when `pinned` is strictly older than `latest`.
@ -76,8 +127,13 @@ def behind(pinned: str, latest: str) -> bool:
# Any number of numeric release components, not three: PyPI stub packages # Any number of numeric release components, not three: PyPI stub packages
# date-stamp a fourth (`types-requests==2.33.0.20260712`), and capping at # date-stamp a fourth (`types-requests==2.33.0.20260712`), and capping at
# three reported an exact pin as unpinned against itself. # three reported an exact pin as unpinned against itself. A project-name
EXACT = re.compile(r"^\s*(?:==\s*)?v?\d+(?:\.\d+)*(?:[-+][0-9A-Za-z.-]+)?\s*$") # prefix (`natural-comparator-1.1`) is part of the pin, since it is what the
# registry's version list spells.
EXACT = re.compile(
r"^\s*(?:==\s*)?(?:[A-Za-z][A-Za-z_.-]*?-(?=v?\d))?"
r"v?\d+(?:\.\d+)*(?:[-+][0-9A-Za-z.-]+)?\s*$"
)
def exact_pin(constraint: str) -> str | None: def exact_pin(constraint: str) -> str | None:
@ -92,4 +148,4 @@ def exact_pin(constraint: str) -> str | None:
text = str(constraint).strip() text = str(constraint).strip()
if not EXACT.match(text): if not EXACT.match(text):
return None return None
return text.lstrip("=").strip().lstrip("v") return _canonical(text.lstrip("=").strip())