mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-20 13:12:20 +01:00
Split wsg_grabber/net.py under the 250-line cap
net.py spoke HTTP for two very different things. The media transfer moves out: _download.py Transfer, download, _stream, _over_budget, _write_body _reject.py classify() -- status line and headers to an Outcome net.py keeps build_session and get_json and re-exports the transfer API through __all__, so downloader.py and test_net.py keep using net.download / net.Transfer / net.digest_of / net.size_of untouched. digest_of had to move down into _download rather than stay put: _stream needs it, and importing it back up from net was a real runtime cycle. net re-exports it. _rejection became classify because it is now imported across a module boundary. Note for the next split: ruff's F401 autofix stripped 11 dead imports here in one pass. __all__ is what stops it doing the same to a re-export. net.py 335 -> 109, _download.py 205, _reject.py 96. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
This commit is contained in:
parent
62ad1484a6
commit
262ef38efe
@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"intent": "Bring python_pkg/wsg_grabber/net.py under the 250-line file cap by separating the media-transfer machinery from the JSON/session half, keeping net.download, net.Transfer, net.digest_of and net.size_of reachable at their existing paths.",
|
||||||
|
"scope": [
|
||||||
|
"python_pkg/wsg_grabber/net.py",
|
||||||
|
"python_pkg/wsg_grabber/_download.py (new)",
|
||||||
|
"python_pkg/wsg_grabber/_reject.py (new)",
|
||||||
|
"Non-goal: no behaviour change, no test changes, no change to retry/resume semantics"
|
||||||
|
],
|
||||||
|
"changes": [
|
||||||
|
"Moved the Transfer dataclass, download(), _stream(), _over_budget() and _write_body() into _download.py; net.py keeps build_session() and get_json().",
|
||||||
|
"Moved the HTTP status classifier into _reject.py and renamed it _rejection -> classify, since it is now imported across a module boundary and a leading underscore would be a private-name import.",
|
||||||
|
"Moved digest_of() down into _download.py because _stream() needs it: leaving it in net.py while _download imported it created a genuine runtime import cycle. net.py re-exports it, so net.digest_of still works.",
|
||||||
|
"size_of() lives in _reject.py and is re-exported through _download and net, so all three existing call paths resolve.",
|
||||||
|
"net.py re-exports Transfer/digest_of/download/size_of through __all__.",
|
||||||
|
"net.py 335 -> 109 lines; _download.py 205; _reject.py 96. All three under the cap."
|
||||||
|
],
|
||||||
|
"verification": [
|
||||||
|
{
|
||||||
|
"command": "PYTHONPATH=. python3 -c \"from python_pkg.wsg_grabber import net; [getattr(net, n) for n in net.__all__]; print(net.digest_of(b''))\"",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "All six __all__ names resolve and net.digest_of(b'') returns 1B2M2Y8AsgTpgAmY7PhCfg==, the known base64 MD5 of the empty string. This also proves the net -> _download -> _reject import chain has no runtime cycle."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "pre-commit run --files python_pkg/wsg_grabber/net.py python_pkg/wsg_grabber/_download.py python_pkg/wsg_grabber/_reject.py",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "Clean on the second run with no output after filtering Passed/Skipped. The first run had ruff autofix 15 dead imports left behind by the move (F401/I001/TC005); the __all__ entries protected the re-exports from that same autofix, and net.__all__ was re-verified as resolving afterwards."
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"risks": [
|
||||||
|
"ruff's F401 autofix removed 11 imports from net.py in one pass. Anything re-exported must be listed in __all__ or it would be deleted silently; verified by resolving net.__all__ after the autofix ran.",
|
||||||
|
"_reject.py type-references Transfer from _download while _download imports classify from _reject; this is only non-circular because the Transfer reference sits under TYPE_CHECKING."
|
||||||
|
],
|
||||||
|
"rollback": [
|
||||||
|
"git revert the commit; net.py returns to its 335-line form and _download.py / _reject.py are deleted.",
|
||||||
|
"After rollback, run python3 -m pytest python_pkg/wsg_grabber/tests/ -q to confirm the suite still passes."
|
||||||
|
]
|
||||||
|
}
|
||||||
205
python_pkg/wsg_grabber/_download.py
Normal file
205
python_pkg/wsg_grabber/_download.py
Normal file
@ -0,0 +1,205 @@
|
|||||||
|
"""Streaming one media file to disk, with resume and verification.
|
||||||
|
|
||||||
|
Split out of :mod:`python_pkg.wsg_grabber.net` to keep that module under the
|
||||||
|
250-line cap. ``net`` still owns the session and the JSON endpoints; everything
|
||||||
|
about pulling bytes from ``i.4cdn.org`` and deciding whether to trust them
|
||||||
|
lives here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from python_pkg.wsg_grabber._reject import classify, size_of
|
||||||
|
from python_pkg.wsg_grabber.constants import (
|
||||||
|
CHUNK_BYTES,
|
||||||
|
CONNECT_TIMEOUT_S,
|
||||||
|
MEDIA_HEADERS,
|
||||||
|
READ_TIMEOUT_S,
|
||||||
|
SIZE_SLACK_BYTES,
|
||||||
|
)
|
||||||
|
from python_pkg.wsg_grabber.models import DownloadResult, Outcome
|
||||||
|
from python_pkg.wsg_grabber.scanner import resume_plan
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_TIMEOUT = (CONNECT_TIMEOUT_S, READ_TIMEOUT_S)
|
||||||
|
_OK = 200
|
||||||
|
_PARTIAL = 206
|
||||||
|
_NOT_FOUND = 404
|
||||||
|
_GONE = 410
|
||||||
|
_RANGE_NOT_SATISFIABLE = 416
|
||||||
|
_TOO_MANY = 429
|
||||||
|
_SERVER_ERROR = 500
|
||||||
|
|
||||||
|
|
||||||
|
def digest_of(payload: bytes) -> str:
|
||||||
|
"""Return the base64 MD5 the API uses to identify a file.
|
||||||
|
|
||||||
|
MD5 here is 4chan's content identifier, not a security control, hence
|
||||||
|
``usedforsecurity=False``.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
payload: Full file contents.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: 24-character base64 digest.
|
||||||
|
"""
|
||||||
|
digest = hashlib.md5(payload, usedforsecurity=False).digest()
|
||||||
|
return base64.b64encode(digest).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Transfer:
|
||||||
|
"""Everything one download needs, bundled to stay within the arg limit."""
|
||||||
|
|
||||||
|
url: str
|
||||||
|
part_path: Path
|
||||||
|
expected_md5: str
|
||||||
|
expected_size: int
|
||||||
|
should_stop: Callable[[], bool]
|
||||||
|
on_progress: Callable[[int], None] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def download(session: requests.Session, transfer: Transfer) -> DownloadResult:
|
||||||
|
"""Stream a file to disk, resuming where possible and verifying the result.
|
||||||
|
|
||||||
|
The bytes are checked against the API's md5 before the download counts as
|
||||||
|
finished, so a truncated or mangled transfer can never reach the reviewer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
session: Shared session.
|
||||||
|
transfer: What to fetch and where to put it.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
DownloadResult: Outcome plus the byte count now on disk.
|
||||||
|
"""
|
||||||
|
plan = resume_plan(size_of(transfer.part_path), transfer.expected_size)
|
||||||
|
if plan.discard:
|
||||||
|
transfer.part_path.unlink(missing_ok=True)
|
||||||
|
headers = dict(MEDIA_HEADERS)
|
||||||
|
if plan.offset:
|
||||||
|
headers["Range"] = f"bytes={plan.offset}-"
|
||||||
|
|
||||||
|
try:
|
||||||
|
return _stream(session, transfer, headers, plan.offset)
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.TRANSIENT,
|
||||||
|
bytes_done=size_of(transfer.part_path),
|
||||||
|
message=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _stream(
|
||||||
|
session: requests.Session,
|
||||||
|
transfer: Transfer,
|
||||||
|
headers: dict[str, str],
|
||||||
|
offset: int,
|
||||||
|
) -> DownloadResult:
|
||||||
|
"""Perform the request and write the body out.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
session: Shared session.
|
||||||
|
transfer: What to fetch and where to put it.
|
||||||
|
headers: Media headers, possibly carrying a ``Range``.
|
||||||
|
offset: Byte offset requested; zero for a fresh download.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
DownloadResult: Outcome plus the byte count now on disk.
|
||||||
|
"""
|
||||||
|
response = session.get(
|
||||||
|
transfer.url,
|
||||||
|
headers=headers,
|
||||||
|
timeout=_TIMEOUT,
|
||||||
|
stream=True,
|
||||||
|
)
|
||||||
|
rejection = classify(response, transfer)
|
||||||
|
if rejection is not None:
|
||||||
|
return rejection
|
||||||
|
|
||||||
|
# A server that ignores Range replies 200 with the whole file, so anything
|
||||||
|
# already on disk has to be thrown away rather than appended to.
|
||||||
|
append = offset > 0 and response.status_code == _PARTIAL
|
||||||
|
written = _write_body(response, transfer, append=append)
|
||||||
|
if written is None:
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.ABORTED,
|
||||||
|
bytes_done=size_of(transfer.part_path),
|
||||||
|
)
|
||||||
|
if _over_budget(written, transfer.expected_size):
|
||||||
|
transfer.part_path.unlink(missing_ok=True)
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.TRANSIENT,
|
||||||
|
bytes_done=0,
|
||||||
|
message=f"body exceeded the advertised {transfer.expected_size} bytes",
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = transfer.part_path.read_bytes()
|
||||||
|
if digest_of(payload) != transfer.expected_md5:
|
||||||
|
transfer.part_path.unlink(missing_ok=True)
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.CHECKSUM_MISMATCH,
|
||||||
|
bytes_done=0,
|
||||||
|
message="md5 did not match the value published by the API",
|
||||||
|
)
|
||||||
|
return DownloadResult(outcome=Outcome.COMPLETED, bytes_done=len(payload))
|
||||||
|
|
||||||
|
|
||||||
|
def _over_budget(written: int, expected: int) -> bool:
|
||||||
|
"""Report whether a transfer outgrew the size the API advertised.
|
||||||
|
|
||||||
|
Without this a server that lies about Content-Length can write until the
|
||||||
|
disk is full; the md5 check alone does not help, because whoever serves the
|
||||||
|
bytes also publishes the digest.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
written: Bytes now on disk.
|
||||||
|
expected: Size the API declared; zero means it did not say.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bool: True when the transfer should be abandoned.
|
||||||
|
"""
|
||||||
|
if expected <= 0:
|
||||||
|
return False
|
||||||
|
return written > expected + SIZE_SLACK_BYTES
|
||||||
|
|
||||||
|
|
||||||
|
def _write_body(
|
||||||
|
response: requests.Response,
|
||||||
|
transfer: Transfer,
|
||||||
|
*,
|
||||||
|
append: bool,
|
||||||
|
) -> int | None:
|
||||||
|
"""Write the response body to the part file.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
response: Streamed response.
|
||||||
|
transfer: What is being fetched.
|
||||||
|
append: Whether to continue an existing part file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int | None: Bytes written, or None when shutdown interrupted it.
|
||||||
|
"""
|
||||||
|
mode = "ab" if append else "wb"
|
||||||
|
done = size_of(transfer.part_path) if append else 0
|
||||||
|
with transfer.part_path.open(mode) as handle:
|
||||||
|
for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
|
||||||
|
if transfer.should_stop():
|
||||||
|
return None
|
||||||
|
if not chunk:
|
||||||
|
continue
|
||||||
|
handle.write(chunk)
|
||||||
|
done += len(chunk)
|
||||||
|
if transfer.on_progress is not None:
|
||||||
|
transfer.on_progress(done)
|
||||||
|
if _over_budget(done, transfer.expected_size):
|
||||||
|
return done
|
||||||
|
return done
|
||||||
96
python_pkg/wsg_grabber/_reject.py
Normal file
96
python_pkg/wsg_grabber/_reject.py
Normal file
@ -0,0 +1,96 @@
|
|||||||
|
"""Deciding whether a media response can be trusted at all.
|
||||||
|
|
||||||
|
Split out of :mod:`python_pkg.wsg_grabber._download` to keep it under the
|
||||||
|
250-line cap. This is pure classification of a status line and headers into an
|
||||||
|
:class:`Outcome`; it never touches the body.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from python_pkg.wsg_grabber.models import DownloadResult, Outcome
|
||||||
|
from python_pkg.wsg_grabber.scanner import parse_retry_after
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from python_pkg.wsg_grabber._download import Transfer
|
||||||
|
|
||||||
|
_OK = 200
|
||||||
|
_PARTIAL = 206
|
||||||
|
_NOT_FOUND = 404
|
||||||
|
_GONE = 410
|
||||||
|
_RANGE_NOT_SATISFIABLE = 416
|
||||||
|
_TOO_MANY = 429
|
||||||
|
_SERVER_ERROR = 500
|
||||||
|
|
||||||
|
|
||||||
|
def size_of(path: Path) -> int:
|
||||||
|
"""Return the size of *path*, or zero when it does not exist.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path: File to measure.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: Size in bytes.
|
||||||
|
"""
|
||||||
|
return path.stat().st_size if path.exists() else 0
|
||||||
|
|
||||||
|
|
||||||
|
def classify(
|
||||||
|
response: requests.Response,
|
||||||
|
transfer: Transfer,
|
||||||
|
) -> DownloadResult | None:
|
||||||
|
"""Classify a non-success status, if any.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
response: The (streamed) response.
|
||||||
|
transfer: The transfer being attempted.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
DownloadResult | None: None when the response is usable.
|
||||||
|
"""
|
||||||
|
status = response.status_code
|
||||||
|
if status in (_NOT_FOUND, _GONE):
|
||||||
|
transfer.part_path.unlink(missing_ok=True)
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.NOT_FOUND,
|
||||||
|
bytes_done=0,
|
||||||
|
message=f"HTTP {status}",
|
||||||
|
)
|
||||||
|
encoding = response.headers.get("Content-Encoding", "identity")
|
||||||
|
if encoding != "identity":
|
||||||
|
# We asked for identity. A compressed body means the far side is not
|
||||||
|
# playing by the rules, and requests would silently expand it -- a
|
||||||
|
# 200 KB response can decode to 200 MB, and the md5 would still match
|
||||||
|
# because the same party published it.
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.TRANSIENT,
|
||||||
|
bytes_done=size_of(transfer.part_path),
|
||||||
|
message=f"refused Content-Encoding: {encoding}",
|
||||||
|
)
|
||||||
|
if status == _RANGE_NOT_SATISFIABLE:
|
||||||
|
# The leftover part is longer than the file actually is; start over.
|
||||||
|
transfer.part_path.unlink(missing_ok=True)
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.TRANSIENT,
|
||||||
|
bytes_done=0,
|
||||||
|
message="range rejected; restarting",
|
||||||
|
)
|
||||||
|
if status == _TOO_MANY or status >= _SERVER_ERROR:
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.TRANSIENT,
|
||||||
|
bytes_done=size_of(transfer.part_path),
|
||||||
|
retry_after=parse_retry_after(response.headers.get("Retry-After")),
|
||||||
|
message=f"HTTP {status}",
|
||||||
|
)
|
||||||
|
if status not in (_OK, _PARTIAL):
|
||||||
|
return DownloadResult(
|
||||||
|
outcome=Outcome.TRANSIENT,
|
||||||
|
bytes_done=size_of(transfer.part_path),
|
||||||
|
message=f"unexpected HTTP {status}",
|
||||||
|
)
|
||||||
|
return None
|
||||||
@ -10,29 +10,36 @@ merely convenient.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import hashlib
|
|
||||||
import json
|
import json
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
# Re-exported through __all__ so net.download / net.Transfer / net.digest_of /
|
||||||
|
# net.size_of keep working for downloader.py and test_net.py; the transfer
|
||||||
|
# machinery lives in _download to keep this module under the 250-line cap.
|
||||||
|
from python_pkg.wsg_grabber._download import (
|
||||||
|
Transfer,
|
||||||
|
digest_of,
|
||||||
|
download,
|
||||||
|
size_of,
|
||||||
|
)
|
||||||
from python_pkg.wsg_grabber.constants import (
|
from python_pkg.wsg_grabber.constants import (
|
||||||
API_HEADERS,
|
API_HEADERS,
|
||||||
CHUNK_BYTES,
|
|
||||||
CONNECT_TIMEOUT_S,
|
CONNECT_TIMEOUT_S,
|
||||||
MAX_JSON_BYTES,
|
MAX_JSON_BYTES,
|
||||||
MEDIA_HEADERS,
|
|
||||||
READ_TIMEOUT_S,
|
READ_TIMEOUT_S,
|
||||||
SIZE_SLACK_BYTES,
|
|
||||||
)
|
)
|
||||||
from python_pkg.wsg_grabber.models import DownloadResult, JsonResponse, Outcome
|
from python_pkg.wsg_grabber.models import JsonResponse
|
||||||
from python_pkg.wsg_grabber.scanner import parse_retry_after, resume_plan
|
|
||||||
|
__all__ = [
|
||||||
|
"Transfer",
|
||||||
|
"build_session",
|
||||||
|
"digest_of",
|
||||||
|
"download",
|
||||||
|
"get_json",
|
||||||
|
"size_of",
|
||||||
|
]
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from collections.abc import Callable
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
_TIMEOUT = (CONNECT_TIMEOUT_S, READ_TIMEOUT_S)
|
_TIMEOUT = (CONNECT_TIMEOUT_S, READ_TIMEOUT_S)
|
||||||
_OK = 200
|
_OK = 200
|
||||||
@ -45,18 +52,6 @@ _TOO_MANY = 429
|
|||||||
_SERVER_ERROR = 500
|
_SERVER_ERROR = 500
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class Transfer:
|
|
||||||
"""Everything one download needs, bundled to stay within the arg limit."""
|
|
||||||
|
|
||||||
url: str
|
|
||||||
part_path: Path
|
|
||||||
expected_md5: str
|
|
||||||
expected_size: int
|
|
||||||
should_stop: Callable[[], bool]
|
|
||||||
on_progress: Callable[[int], None] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def build_session() -> requests.Session:
|
def build_session() -> requests.Session:
|
||||||
"""Return a session configured for the API host.
|
"""Return a session configured for the API host.
|
||||||
|
|
||||||
@ -68,22 +63,6 @@ def build_session() -> requests.Session:
|
|||||||
return session
|
return session
|
||||||
|
|
||||||
|
|
||||||
def digest_of(payload: bytes) -> str:
|
|
||||||
"""Return the base64 MD5 the API uses to identify a file.
|
|
||||||
|
|
||||||
MD5 here is 4chan's content identifier, not a security control, hence
|
|
||||||
``usedforsecurity=False``.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
payload: Full file contents.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: 24-character base64 digest.
|
|
||||||
"""
|
|
||||||
digest = hashlib.md5(payload, usedforsecurity=False).digest()
|
|
||||||
return base64.b64encode(digest).decode("ascii")
|
|
||||||
|
|
||||||
|
|
||||||
def get_json(
|
def get_json(
|
||||||
session: requests.Session,
|
session: requests.Session,
|
||||||
url: str,
|
url: str,
|
||||||
@ -128,208 +107,3 @@ def get_json(
|
|||||||
not_modified=False,
|
not_modified=False,
|
||||||
not_found=False,
|
not_found=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def download(session: requests.Session, transfer: Transfer) -> DownloadResult:
|
|
||||||
"""Stream a file to disk, resuming where possible and verifying the result.
|
|
||||||
|
|
||||||
The bytes are checked against the API's md5 before the download counts as
|
|
||||||
finished, so a truncated or mangled transfer can never reach the reviewer.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
session: Shared session.
|
|
||||||
transfer: What to fetch and where to put it.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
DownloadResult: Outcome plus the byte count now on disk.
|
|
||||||
"""
|
|
||||||
plan = resume_plan(size_of(transfer.part_path), transfer.expected_size)
|
|
||||||
if plan.discard:
|
|
||||||
transfer.part_path.unlink(missing_ok=True)
|
|
||||||
headers = dict(MEDIA_HEADERS)
|
|
||||||
if plan.offset:
|
|
||||||
headers["Range"] = f"bytes={plan.offset}-"
|
|
||||||
|
|
||||||
try:
|
|
||||||
return _stream(session, transfer, headers, plan.offset)
|
|
||||||
except requests.RequestException as exc:
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.TRANSIENT,
|
|
||||||
bytes_done=size_of(transfer.part_path),
|
|
||||||
message=str(exc),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _stream(
|
|
||||||
session: requests.Session,
|
|
||||||
transfer: Transfer,
|
|
||||||
headers: dict[str, str],
|
|
||||||
offset: int,
|
|
||||||
) -> DownloadResult:
|
|
||||||
"""Perform the request and write the body out.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
session: Shared session.
|
|
||||||
transfer: What to fetch and where to put it.
|
|
||||||
headers: Media headers, possibly carrying a ``Range``.
|
|
||||||
offset: Byte offset requested; zero for a fresh download.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
DownloadResult: Outcome plus the byte count now on disk.
|
|
||||||
"""
|
|
||||||
response = session.get(
|
|
||||||
transfer.url,
|
|
||||||
headers=headers,
|
|
||||||
timeout=_TIMEOUT,
|
|
||||||
stream=True,
|
|
||||||
)
|
|
||||||
rejection = _rejection(response, transfer)
|
|
||||||
if rejection is not None:
|
|
||||||
return rejection
|
|
||||||
|
|
||||||
# A server that ignores Range replies 200 with the whole file, so anything
|
|
||||||
# already on disk has to be thrown away rather than appended to.
|
|
||||||
append = offset > 0 and response.status_code == _PARTIAL
|
|
||||||
written = _write_body(response, transfer, append=append)
|
|
||||||
if written is None:
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.ABORTED,
|
|
||||||
bytes_done=size_of(transfer.part_path),
|
|
||||||
)
|
|
||||||
if _over_budget(written, transfer.expected_size):
|
|
||||||
transfer.part_path.unlink(missing_ok=True)
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.TRANSIENT,
|
|
||||||
bytes_done=0,
|
|
||||||
message=f"body exceeded the advertised {transfer.expected_size} bytes",
|
|
||||||
)
|
|
||||||
|
|
||||||
payload = transfer.part_path.read_bytes()
|
|
||||||
if digest_of(payload) != transfer.expected_md5:
|
|
||||||
transfer.part_path.unlink(missing_ok=True)
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.CHECKSUM_MISMATCH,
|
|
||||||
bytes_done=0,
|
|
||||||
message="md5 did not match the value published by the API",
|
|
||||||
)
|
|
||||||
return DownloadResult(outcome=Outcome.COMPLETED, bytes_done=len(payload))
|
|
||||||
|
|
||||||
|
|
||||||
def _rejection(
|
|
||||||
response: requests.Response,
|
|
||||||
transfer: Transfer,
|
|
||||||
) -> DownloadResult | None:
|
|
||||||
"""Classify a non-success status, if any.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
response: The (streamed) response.
|
|
||||||
transfer: The transfer being attempted.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
DownloadResult | None: None when the response is usable.
|
|
||||||
"""
|
|
||||||
status = response.status_code
|
|
||||||
if status in (_NOT_FOUND, _GONE):
|
|
||||||
transfer.part_path.unlink(missing_ok=True)
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.NOT_FOUND,
|
|
||||||
bytes_done=0,
|
|
||||||
message=f"HTTP {status}",
|
|
||||||
)
|
|
||||||
encoding = response.headers.get("Content-Encoding", "identity")
|
|
||||||
if encoding != "identity":
|
|
||||||
# We asked for identity. A compressed body means the far side is not
|
|
||||||
# playing by the rules, and requests would silently expand it -- a
|
|
||||||
# 200 KB response can decode to 200 MB, and the md5 would still match
|
|
||||||
# because the same party published it.
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.TRANSIENT,
|
|
||||||
bytes_done=size_of(transfer.part_path),
|
|
||||||
message=f"refused Content-Encoding: {encoding}",
|
|
||||||
)
|
|
||||||
if status == _RANGE_NOT_SATISFIABLE:
|
|
||||||
# The leftover part is longer than the file actually is; start over.
|
|
||||||
transfer.part_path.unlink(missing_ok=True)
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.TRANSIENT,
|
|
||||||
bytes_done=0,
|
|
||||||
message="range rejected; restarting",
|
|
||||||
)
|
|
||||||
if status == _TOO_MANY or status >= _SERVER_ERROR:
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.TRANSIENT,
|
|
||||||
bytes_done=size_of(transfer.part_path),
|
|
||||||
retry_after=parse_retry_after(response.headers.get("Retry-After")),
|
|
||||||
message=f"HTTP {status}",
|
|
||||||
)
|
|
||||||
if status not in (_OK, _PARTIAL):
|
|
||||||
return DownloadResult(
|
|
||||||
outcome=Outcome.TRANSIENT,
|
|
||||||
bytes_done=size_of(transfer.part_path),
|
|
||||||
message=f"unexpected HTTP {status}",
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _over_budget(written: int, expected: int) -> bool:
|
|
||||||
"""Report whether a transfer outgrew the size the API advertised.
|
|
||||||
|
|
||||||
Without this a server that lies about Content-Length can write until the
|
|
||||||
disk is full; the md5 check alone does not help, because whoever serves the
|
|
||||||
bytes also publishes the digest.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
written: Bytes now on disk.
|
|
||||||
expected: Size the API declared; zero means it did not say.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
bool: True when the transfer should be abandoned.
|
|
||||||
"""
|
|
||||||
if expected <= 0:
|
|
||||||
return False
|
|
||||||
return written > expected + SIZE_SLACK_BYTES
|
|
||||||
|
|
||||||
|
|
||||||
def _write_body(
|
|
||||||
response: requests.Response,
|
|
||||||
transfer: Transfer,
|
|
||||||
*,
|
|
||||||
append: bool,
|
|
||||||
) -> int | None:
|
|
||||||
"""Write the response body to the part file.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
response: Streamed response.
|
|
||||||
transfer: What is being fetched.
|
|
||||||
append: Whether to continue an existing part file.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
int | None: Bytes written, or None when shutdown interrupted it.
|
|
||||||
"""
|
|
||||||
mode = "ab" if append else "wb"
|
|
||||||
done = size_of(transfer.part_path) if append else 0
|
|
||||||
with transfer.part_path.open(mode) as handle:
|
|
||||||
for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
|
|
||||||
if transfer.should_stop():
|
|
||||||
return None
|
|
||||||
if not chunk:
|
|
||||||
continue
|
|
||||||
handle.write(chunk)
|
|
||||||
done += len(chunk)
|
|
||||||
if transfer.on_progress is not None:
|
|
||||||
transfer.on_progress(done)
|
|
||||||
if _over_budget(done, transfer.expected_size):
|
|
||||||
return done
|
|
||||||
return done
|
|
||||||
|
|
||||||
|
|
||||||
def size_of(path: Path) -> int:
|
|
||||||
"""Return the size of *path*, or zero when it does not exist.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
path: File to measure.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
int: Size in bytes.
|
|
||||||
"""
|
|
||||||
return path.stat().st_size if path.exists() else 0
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user