mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 22:07:03 +01:00
Krzysztof kuhy Rudnicki
262ef38efe
net.py spoke HTTP for two very different things. The media transfer moves out: _download.py Transfer, download, _stream, _over_budget, _write_body _reject.py classify() -- status line and headers to an Outcome net.py keeps build_session and get_json and re-exports the transfer API through __all__, so downloader.py and test_net.py keep using net.download / net.Transfer / net.digest_of / net.size_of untouched. digest_of had to move down into _download rather than stay put: _stream needs it, and importing it back up from net was a real runtime cycle. net re-exports it. _rejection became classify because it is now imported across a module boundary. Note for the next split: ruff's F401 autofix stripped 11 dead imports here in one pass. __all__ is what stops it doing the same to a re-export. net.py 335 -> 109, _download.py 205, _reject.py 96. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
110 lines
3.0 KiB
Python
110 lines
3.0 KiB
Python
"""The only module that speaks HTTP.
|
|
|
|
Two hosts with different manners. ``a.4cdn.org`` is polite and supports
|
|
``If-Modified-Since``, so a steady-state rescan costs almost nothing.
|
|
``i.4cdn.org`` answered a bare request with 429 during development, and the
|
|
cookies it sets in reply (``__cf_bm``, ``_cfuvid``) live in the session jar --
|
|
which is why one long-lived :class:`requests.Session` is required rather than
|
|
merely convenient.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import requests
|
|
|
|
# Re-exported through __all__ so net.download / net.Transfer / net.digest_of /
|
|
# net.size_of keep working for downloader.py and test_net.py; the transfer
|
|
# machinery lives in _download to keep this module under the 250-line cap.
|
|
from python_pkg.wsg_grabber._download import (
|
|
Transfer,
|
|
digest_of,
|
|
download,
|
|
size_of,
|
|
)
|
|
from python_pkg.wsg_grabber.constants import (
|
|
API_HEADERS,
|
|
CONNECT_TIMEOUT_S,
|
|
MAX_JSON_BYTES,
|
|
READ_TIMEOUT_S,
|
|
)
|
|
from python_pkg.wsg_grabber.models import JsonResponse
|
|
|
|
__all__ = [
|
|
"Transfer",
|
|
"build_session",
|
|
"digest_of",
|
|
"download",
|
|
"get_json",
|
|
"size_of",
|
|
]
|
|
|
|
|
|
_TIMEOUT = (CONNECT_TIMEOUT_S, READ_TIMEOUT_S)
|
|
_OK = 200
|
|
_PARTIAL = 206
|
|
_NOT_MODIFIED = 304
|
|
_NOT_FOUND = 404
|
|
_GONE = 410
|
|
_RANGE_NOT_SATISFIABLE = 416
|
|
_TOO_MANY = 429
|
|
_SERVER_ERROR = 500
|
|
|
|
|
|
def build_session() -> requests.Session:
|
|
"""Return a session configured for the API host.
|
|
|
|
Returns:
|
|
requests.Session: Caller owns it and must close it.
|
|
"""
|
|
session = requests.Session()
|
|
session.headers.update(API_HEADERS)
|
|
return session
|
|
|
|
|
|
def get_json(
|
|
session: requests.Session,
|
|
url: str,
|
|
last_modified: str | None = None,
|
|
) -> JsonResponse:
|
|
"""Fetch and decode a JSON endpoint, conditionally when possible.
|
|
|
|
Args:
|
|
session: Shared session.
|
|
url: Absolute API URL.
|
|
last_modified: Previously seen ``Last-Modified`` value, echoed back as
|
|
``If-Modified-Since`` so an unchanged thread costs a 304.
|
|
|
|
Returns:
|
|
JsonResponse: Decoded payload, or a flag explaining why there is none.
|
|
"""
|
|
headers = {"If-Modified-Since": last_modified} if last_modified else {}
|
|
response = session.get(url, headers=headers, timeout=_TIMEOUT)
|
|
status = response.status_code
|
|
if status == _NOT_MODIFIED:
|
|
return JsonResponse(
|
|
payload=None,
|
|
last_modified=last_modified,
|
|
not_modified=True,
|
|
not_found=False,
|
|
)
|
|
if status in (_NOT_FOUND, _GONE):
|
|
return JsonResponse(
|
|
payload=None,
|
|
last_modified=None,
|
|
not_modified=False,
|
|
not_found=True,
|
|
)
|
|
response.raise_for_status()
|
|
body = response.content[: MAX_JSON_BYTES + 1]
|
|
if len(body) > MAX_JSON_BYTES:
|
|
msg = f"API response exceeded {MAX_JSON_BYTES} bytes"
|
|
raise ValueError(msg)
|
|
return JsonResponse(
|
|
payload=json.loads(body),
|
|
last_modified=response.headers.get("Last-Modified"),
|
|
not_modified=False,
|
|
not_found=False,
|
|
)
|