utils/agent_ready/backends.py

"""Two backends behind one ``get(path)``: a build directory and a live site.

DirSite answers like a plain static host (index.html for ``/x/``, a 404 for
anything missing), so the find/read checks are written once for both modes.
"""

from __future__ import annotations

import http.client
import threading
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
from urllib.parse import unquote, urljoin, urlsplit

from agent_ready.report import Undeterminable

DEFAULT_UA = "Claude-User/1.0 (+agentready-check)"
MAX_BODY = 5 * 1024 * 1024
_EXT_MIME = {
    ".html": "text/html",
    ".md": "text/markdown",
    ".txt": "text/plain",
    ".xml": "application/xml",
    ".json": "application/json",
}


@dataclass(frozen=True)
class Response:
    """A fetched resource: final status, bare media type, headers, body."""

    status: int
    ctype: str
    body: bytes
    headers: dict[str, str] = field(default_factory=dict)

    @property
    def text(self) -> str:
        """Body decoded as UTF-8 (lenient)."""
        return self.body.decode("utf-8", "replace")


class Site:
    """Common URL handling; subclasses implement ``_fetch``."""

    live = False

    def __init__(self, origin: str) -> None:
        self.origin = origin.rstrip("/")
        self._cache: dict[tuple[str, str], Response | Undeterminable] = {}
        self._lock = threading.Lock()

    def to_path(self, url: str, base: str = "/") -> str | None:
        """Site path for a URL or relative href; None when off-site."""
        absolute = urljoin(self.origin + base, url.strip())
        parts = urlsplit(absolute)
        if (
            parts.scheme not in ("http", "https")
            or parts.netloc != urlsplit(self.origin).netloc
        ):
            return None
        return parts.path or "/"

    def get(self, path: str, ua: str = DEFAULT_UA) -> Response:
        """Fetch ``path`` (memoized, including network failures)."""
        key = (path, ua)
        with self._lock:
            hit = self._cache.get(key)
        if hit is None:
            try:
                hit = self._fetch(path, ua)
            except Undeterminable as exc:
                hit = exc
            with self._lock:
                self._cache[key] = hit
        if isinstance(hit, Undeterminable):
            raise hit
        return hit

    def prefetch(self, paths: list[str], uas: tuple[str, ...] = (DEFAULT_UA,)) -> None:
        """Warm the cache for every (path, ua) pair; parallel for live sites."""
        if not self.live:
            return
        pairs = [(p, ua) for p in dict.fromkeys(paths) for ua in uas]
        with ThreadPoolExecutor(max_workers=8) as pool:
            list(pool.map(lambda pair: self._safe_get(*pair), pairs))

    def _safe_get(self, path: str, ua: str) -> None:
        try:
            self.get(path, ua)
        except Undeterminable:
            pass

    def _fetch(self, path: str, ua: str) -> Response:
        raise NotImplementedError


class DirSite(Site):
    """A build output directory served the way a static host serves it."""

    def __init__(self, root: Path) -> None:
        super().__init__("https://dist.invalid")
        self.root = root.resolve()

    def file_for(self, path: str) -> Path | None:
        """The file a static host would serve for ``path``."""
        rel = unquote(path.split("?", 1)[0]).lstrip("/")
        base = (self.root / rel).resolve()
        if not base.is_relative_to(self.root):
            return None
        cands = (
            [base / "index.html"]
            if path.endswith("/") or not rel
            else [base, base.with_name(base.name + ".html"), base / "index.html"]
        )
        return next((c for c in cands if c.is_file()), None)

    def _fetch(self, path: str, ua: str) -> Response:
        target = self.file_for(path)
        if target is None:
            fallback = self.root / "404.html"
            body = fallback.read_bytes() if fallback.is_file() else b""
            return Response(404, "text/html", body)
        return Response(
            200,
            _EXT_MIME.get(target.suffix, "application/octet-stream"),
            target.read_bytes(),
        )


class HttpSite(Site):
    """A live site. Follows redirects; HTTP errors are responses, not raises."""

    live = True

    def __init__(self, base: str, timeout: float = 8.0, budget: float = 45.0) -> None:
        parts = urlsplit(base)
        super().__init__(f"{parts.scheme}://{parts.netloc}")
        self.timeout = timeout
        self.deadline = time.monotonic() + budget

    def _remaining(self) -> float:
        left = self.deadline - time.monotonic()
        if left < 1:
            raise Undeterminable("overall time budget used up")
        return min(self.timeout, left)

    def _fetch(self, path: str, ua: str) -> Response:
        return self.request(path, ua)

    def request(
        self,
        path: str,
        ua: str = DEFAULT_UA,
        data: bytes | None = None,
        headers: dict[str, str] | None = None,
    ) -> Response:
        """Uncached GET (or POST when ``data`` is given)."""
        req = urllib.request.Request(
            self.origin + path,
            data=data,
            headers={"User-Agent": ua, "Accept": "*/*", **(headers or {})},
        )
        try:
            with urllib.request.urlopen(req, timeout=self._remaining()) as resp:
                return _response(resp.status, resp.headers, resp.read(MAX_BODY))
        except urllib.error.HTTPError as exc:
            return _response(exc.code, exc.headers, exc.read(MAX_BODY))
        except (
            urllib.error.URLError,
            TimeoutError,
            OSError,
            http.client.HTTPException,
        ) as exc:
            raise Undeterminable(f"{path}: {getattr(exc, 'reason', exc)}") from exc


def _response(status: int, headers: http.client.HTTPMessage, body: bytes) -> Response:
    """Normalise headers: lowercase keys, repeated headers joined by ', '."""
    flat: dict[str, str] = {}
    for key in set(headers.keys()):
        flat[key.lower()] = ", ".join(headers.get_all(key) or [])
    ctype = flat.get("content-type", "").split(";", 1)[0].strip().lower()
    return Response(status, ctype, body, flat)