diff --git a/docs/superpowers/evidence/wsg-grabber-net-split-2026-08-16.json b/docs/superpowers/evidence/wsg-grabber-net-split-2026-08-16.json new file mode 100644 index 00000000..e6830631 --- /dev/null +++ b/docs/superpowers/evidence/wsg-grabber-net-split-2026-08-16.json @@ -0,0 +1,37 @@ +{ + "intent": "Bring python_pkg/wsg_grabber/net.py under the 250-line file cap by separating the media-transfer machinery from the JSON/session half, keeping net.download, net.Transfer, net.digest_of and net.size_of reachable at their existing paths.", + "scope": [ + "python_pkg/wsg_grabber/net.py", + "python_pkg/wsg_grabber/_download.py (new)", + "python_pkg/wsg_grabber/_reject.py (new)", + "Non-goal: no behaviour change, no test changes, no change to retry/resume semantics" + ], + "changes": [ + "Moved the Transfer dataclass, download(), _stream(), _over_budget() and _write_body() into _download.py; net.py keeps build_session() and get_json().", + "Moved the HTTP status classifier into _reject.py and renamed it _rejection -> classify, since it is now imported across a module boundary and a leading underscore would be a private-name import.", + "Moved digest_of() down into _download.py because _stream() needs it: leaving it in net.py while _download imported it created a genuine runtime import cycle. net.py re-exports it, so net.digest_of still works.", + "size_of() lives in _reject.py and is re-exported through _download and net, so all three existing call paths resolve.", + "net.py re-exports Transfer/digest_of/download/size_of through __all__.", + "net.py 335 -> 109 lines; _download.py 205; _reject.py 96. All three under the cap." + ], + "verification": [ + { + "command": "PYTHONPATH=. python3 -c \"from python_pkg.wsg_grabber import net; [getattr(net, n) for n in net.__all__]; print(net.digest_of(b''))\"", + "result": "pass", + "evidence": "All six __all__ names resolve and net.digest_of(b'') returns 1B2M2Y8AsgTpgAmY7PhCfg==, the known base64 MD5 of the empty string. This also proves the net -> _download -> _reject import chain has no runtime cycle." + }, + { + "command": "pre-commit run --files python_pkg/wsg_grabber/net.py python_pkg/wsg_grabber/_download.py python_pkg/wsg_grabber/_reject.py", + "result": "pass", + "evidence": "Clean on the second run with no output after filtering Passed/Skipped. The first run had ruff autofix 15 dead imports left behind by the move (F401/I001/TC005); the __all__ entries protected the re-exports from that same autofix, and net.__all__ was re-verified as resolving afterwards." + } + ], + "risks": [ + "ruff's F401 autofix removed 11 imports from net.py in one pass. Anything re-exported must be listed in __all__ or it would be deleted silently; verified by resolving net.__all__ after the autofix ran.", + "_reject.py type-references Transfer from _download while _download imports classify from _reject; this is only non-circular because the Transfer reference sits under TYPE_CHECKING." + ], + "rollback": [ + "git revert the commit; net.py returns to its 335-line form and _download.py / _reject.py are deleted.", + "After rollback, run python3 -m pytest python_pkg/wsg_grabber/tests/ -q to confirm the suite still passes." + ] +} diff --git a/python_pkg/wsg_grabber/_download.py b/python_pkg/wsg_grabber/_download.py new file mode 100644 index 00000000..eea11b54 --- /dev/null +++ b/python_pkg/wsg_grabber/_download.py @@ -0,0 +1,205 @@ +"""Streaming one media file to disk, with resume and verification. + +Split out of :mod:`python_pkg.wsg_grabber.net` to keep that module under the +250-line cap. ``net`` still owns the session and the JSON endpoints; everything +about pulling bytes from ``i.4cdn.org`` and deciding whether to trust them +lives here. +""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass +import hashlib +from typing import TYPE_CHECKING + +import requests + +from python_pkg.wsg_grabber._reject import classify, size_of +from python_pkg.wsg_grabber.constants import ( + CHUNK_BYTES, + CONNECT_TIMEOUT_S, + MEDIA_HEADERS, + READ_TIMEOUT_S, + SIZE_SLACK_BYTES, +) +from python_pkg.wsg_grabber.models import DownloadResult, Outcome +from python_pkg.wsg_grabber.scanner import resume_plan + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + +_TIMEOUT = (CONNECT_TIMEOUT_S, READ_TIMEOUT_S) +_OK = 200 +_PARTIAL = 206 +_NOT_FOUND = 404 +_GONE = 410 +_RANGE_NOT_SATISFIABLE = 416 +_TOO_MANY = 429 +_SERVER_ERROR = 500 + + +def digest_of(payload: bytes) -> str: + """Return the base64 MD5 the API uses to identify a file. + + MD5 here is 4chan's content identifier, not a security control, hence + ``usedforsecurity=False``. + + Args: + payload: Full file contents. + + Returns: + str: 24-character base64 digest. + """ + digest = hashlib.md5(payload, usedforsecurity=False).digest() + return base64.b64encode(digest).decode("ascii") + + +@dataclass(frozen=True, slots=True) +class Transfer: + """Everything one download needs, bundled to stay within the arg limit.""" + + url: str + part_path: Path + expected_md5: str + expected_size: int + should_stop: Callable[[], bool] + on_progress: Callable[[int], None] | None = None + + +def download(session: requests.Session, transfer: Transfer) -> DownloadResult: + """Stream a file to disk, resuming where possible and verifying the result. + + The bytes are checked against the API's md5 before the download counts as + finished, so a truncated or mangled transfer can never reach the reviewer. + + Args: + session: Shared session. + transfer: What to fetch and where to put it. + + Returns: + DownloadResult: Outcome plus the byte count now on disk. + """ + plan = resume_plan(size_of(transfer.part_path), transfer.expected_size) + if plan.discard: + transfer.part_path.unlink(missing_ok=True) + headers = dict(MEDIA_HEADERS) + if plan.offset: + headers["Range"] = f"bytes={plan.offset}-" + + try: + return _stream(session, transfer, headers, plan.offset) + except requests.RequestException as exc: + return DownloadResult( + outcome=Outcome.TRANSIENT, + bytes_done=size_of(transfer.part_path), + message=str(exc), + ) + + +def _stream( + session: requests.Session, + transfer: Transfer, + headers: dict[str, str], + offset: int, +) -> DownloadResult: + """Perform the request and write the body out. + + Args: + session: Shared session. + transfer: What to fetch and where to put it. + headers: Media headers, possibly carrying a ``Range``. + offset: Byte offset requested; zero for a fresh download. + + Returns: + DownloadResult: Outcome plus the byte count now on disk. + """ + response = session.get( + transfer.url, + headers=headers, + timeout=_TIMEOUT, + stream=True, + ) + rejection = classify(response, transfer) + if rejection is not None: + return rejection + + # A server that ignores Range replies 200 with the whole file, so anything + # already on disk has to be thrown away rather than appended to. + append = offset > 0 and response.status_code == _PARTIAL + written = _write_body(response, transfer, append=append) + if written is None: + return DownloadResult( + outcome=Outcome.ABORTED, + bytes_done=size_of(transfer.part_path), + ) + if _over_budget(written, transfer.expected_size): + transfer.part_path.unlink(missing_ok=True) + return DownloadResult( + outcome=Outcome.TRANSIENT, + bytes_done=0, + message=f"body exceeded the advertised {transfer.expected_size} bytes", + ) + + payload = transfer.part_path.read_bytes() + if digest_of(payload) != transfer.expected_md5: + transfer.part_path.unlink(missing_ok=True) + return DownloadResult( + outcome=Outcome.CHECKSUM_MISMATCH, + bytes_done=0, + message="md5 did not match the value published by the API", + ) + return DownloadResult(outcome=Outcome.COMPLETED, bytes_done=len(payload)) + + +def _over_budget(written: int, expected: int) -> bool: + """Report whether a transfer outgrew the size the API advertised. + + Without this a server that lies about Content-Length can write until the + disk is full; the md5 check alone does not help, because whoever serves the + bytes also publishes the digest. + + Args: + written: Bytes now on disk. + expected: Size the API declared; zero means it did not say. + + Returns: + bool: True when the transfer should be abandoned. + """ + if expected <= 0: + return False + return written > expected + SIZE_SLACK_BYTES + + +def _write_body( + response: requests.Response, + transfer: Transfer, + *, + append: bool, +) -> int | None: + """Write the response body to the part file. + + Args: + response: Streamed response. + transfer: What is being fetched. + append: Whether to continue an existing part file. + + Returns: + int | None: Bytes written, or None when shutdown interrupted it. + """ + mode = "ab" if append else "wb" + done = size_of(transfer.part_path) if append else 0 + with transfer.part_path.open(mode) as handle: + for chunk in response.iter_content(chunk_size=CHUNK_BYTES): + if transfer.should_stop(): + return None + if not chunk: + continue + handle.write(chunk) + done += len(chunk) + if transfer.on_progress is not None: + transfer.on_progress(done) + if _over_budget(done, transfer.expected_size): + return done + return done diff --git a/python_pkg/wsg_grabber/_reject.py b/python_pkg/wsg_grabber/_reject.py new file mode 100644 index 00000000..3c4369d2 --- /dev/null +++ b/python_pkg/wsg_grabber/_reject.py @@ -0,0 +1,96 @@ +"""Deciding whether a media response can be trusted at all. + +Split out of :mod:`python_pkg.wsg_grabber._download` to keep it under the +250-line cap. This is pure classification of a status line and headers into an +:class:`Outcome`; it never touches the body. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.wsg_grabber.models import DownloadResult, Outcome +from python_pkg.wsg_grabber.scanner import parse_retry_after + +if TYPE_CHECKING: + from pathlib import Path + + import requests + + from python_pkg.wsg_grabber._download import Transfer + +_OK = 200 +_PARTIAL = 206 +_NOT_FOUND = 404 +_GONE = 410 +_RANGE_NOT_SATISFIABLE = 416 +_TOO_MANY = 429 +_SERVER_ERROR = 500 + + +def size_of(path: Path) -> int: + """Return the size of *path*, or zero when it does not exist. + + Args: + path: File to measure. + + Returns: + int: Size in bytes. + """ + return path.stat().st_size if path.exists() else 0 + + +def classify( + response: requests.Response, + transfer: Transfer, +) -> DownloadResult | None: + """Classify a non-success status, if any. + + Args: + response: The (streamed) response. + transfer: The transfer being attempted. + + Returns: + DownloadResult | None: None when the response is usable. + """ + status = response.status_code + if status in (_NOT_FOUND, _GONE): + transfer.part_path.unlink(missing_ok=True) + return DownloadResult( + outcome=Outcome.NOT_FOUND, + bytes_done=0, + message=f"HTTP {status}", + ) + encoding = response.headers.get("Content-Encoding", "identity") + if encoding != "identity": + # We asked for identity. A compressed body means the far side is not + # playing by the rules, and requests would silently expand it -- a + # 200 KB response can decode to 200 MB, and the md5 would still match + # because the same party published it. + return DownloadResult( + outcome=Outcome.TRANSIENT, + bytes_done=size_of(transfer.part_path), + message=f"refused Content-Encoding: {encoding}", + ) + if status == _RANGE_NOT_SATISFIABLE: + # The leftover part is longer than the file actually is; start over. + transfer.part_path.unlink(missing_ok=True) + return DownloadResult( + outcome=Outcome.TRANSIENT, + bytes_done=0, + message="range rejected; restarting", + ) + if status == _TOO_MANY or status >= _SERVER_ERROR: + return DownloadResult( + outcome=Outcome.TRANSIENT, + bytes_done=size_of(transfer.part_path), + retry_after=parse_retry_after(response.headers.get("Retry-After")), + message=f"HTTP {status}", + ) + if status not in (_OK, _PARTIAL): + return DownloadResult( + outcome=Outcome.TRANSIENT, + bytes_done=size_of(transfer.part_path), + message=f"unexpected HTTP {status}", + ) + return None diff --git a/python_pkg/wsg_grabber/net.py b/python_pkg/wsg_grabber/net.py index d05c80bc..ec08d136 100644 --- a/python_pkg/wsg_grabber/net.py +++ b/python_pkg/wsg_grabber/net.py @@ -10,29 +10,36 @@ merely convenient. from __future__ import annotations -import base64 -from dataclasses import dataclass -import hashlib import json -from typing import TYPE_CHECKING import requests +# Re-exported through __all__ so net.download / net.Transfer / net.digest_of / +# net.size_of keep working for downloader.py and test_net.py; the transfer +# machinery lives in _download to keep this module under the 250-line cap. +from python_pkg.wsg_grabber._download import ( + Transfer, + digest_of, + download, + size_of, +) from python_pkg.wsg_grabber.constants import ( API_HEADERS, - CHUNK_BYTES, CONNECT_TIMEOUT_S, MAX_JSON_BYTES, - MEDIA_HEADERS, READ_TIMEOUT_S, - SIZE_SLACK_BYTES, ) -from python_pkg.wsg_grabber.models import DownloadResult, JsonResponse, Outcome -from python_pkg.wsg_grabber.scanner import parse_retry_after, resume_plan +from python_pkg.wsg_grabber.models import JsonResponse + +__all__ = [ + "Transfer", + "build_session", + "digest_of", + "download", + "get_json", + "size_of", +] -if TYPE_CHECKING: - from collections.abc import Callable - from pathlib import Path _TIMEOUT = (CONNECT_TIMEOUT_S, READ_TIMEOUT_S) _OK = 200 @@ -45,18 +52,6 @@ _TOO_MANY = 429 _SERVER_ERROR = 500 -@dataclass(frozen=True, slots=True) -class Transfer: - """Everything one download needs, bundled to stay within the arg limit.""" - - url: str - part_path: Path - expected_md5: str - expected_size: int - should_stop: Callable[[], bool] - on_progress: Callable[[int], None] | None = None - - def build_session() -> requests.Session: """Return a session configured for the API host. @@ -68,22 +63,6 @@ def build_session() -> requests.Session: return session -def digest_of(payload: bytes) -> str: - """Return the base64 MD5 the API uses to identify a file. - - MD5 here is 4chan's content identifier, not a security control, hence - ``usedforsecurity=False``. - - Args: - payload: Full file contents. - - Returns: - str: 24-character base64 digest. - """ - digest = hashlib.md5(payload, usedforsecurity=False).digest() - return base64.b64encode(digest).decode("ascii") - - def get_json( session: requests.Session, url: str, @@ -128,208 +107,3 @@ def get_json( not_modified=False, not_found=False, ) - - -def download(session: requests.Session, transfer: Transfer) -> DownloadResult: - """Stream a file to disk, resuming where possible and verifying the result. - - The bytes are checked against the API's md5 before the download counts as - finished, so a truncated or mangled transfer can never reach the reviewer. - - Args: - session: Shared session. - transfer: What to fetch and where to put it. - - Returns: - DownloadResult: Outcome plus the byte count now on disk. - """ - plan = resume_plan(size_of(transfer.part_path), transfer.expected_size) - if plan.discard: - transfer.part_path.unlink(missing_ok=True) - headers = dict(MEDIA_HEADERS) - if plan.offset: - headers["Range"] = f"bytes={plan.offset}-" - - try: - return _stream(session, transfer, headers, plan.offset) - except requests.RequestException as exc: - return DownloadResult( - outcome=Outcome.TRANSIENT, - bytes_done=size_of(transfer.part_path), - message=str(exc), - ) - - -def _stream( - session: requests.Session, - transfer: Transfer, - headers: dict[str, str], - offset: int, -) -> DownloadResult: - """Perform the request and write the body out. - - Args: - session: Shared session. - transfer: What to fetch and where to put it. - headers: Media headers, possibly carrying a ``Range``. - offset: Byte offset requested; zero for a fresh download. - - Returns: - DownloadResult: Outcome plus the byte count now on disk. - """ - response = session.get( - transfer.url, - headers=headers, - timeout=_TIMEOUT, - stream=True, - ) - rejection = _rejection(response, transfer) - if rejection is not None: - return rejection - - # A server that ignores Range replies 200 with the whole file, so anything - # already on disk has to be thrown away rather than appended to. - append = offset > 0 and response.status_code == _PARTIAL - written = _write_body(response, transfer, append=append) - if written is None: - return DownloadResult( - outcome=Outcome.ABORTED, - bytes_done=size_of(transfer.part_path), - ) - if _over_budget(written, transfer.expected_size): - transfer.part_path.unlink(missing_ok=True) - return DownloadResult( - outcome=Outcome.TRANSIENT, - bytes_done=0, - message=f"body exceeded the advertised {transfer.expected_size} bytes", - ) - - payload = transfer.part_path.read_bytes() - if digest_of(payload) != transfer.expected_md5: - transfer.part_path.unlink(missing_ok=True) - return DownloadResult( - outcome=Outcome.CHECKSUM_MISMATCH, - bytes_done=0, - message="md5 did not match the value published by the API", - ) - return DownloadResult(outcome=Outcome.COMPLETED, bytes_done=len(payload)) - - -def _rejection( - response: requests.Response, - transfer: Transfer, -) -> DownloadResult | None: - """Classify a non-success status, if any. - - Args: - response: The (streamed) response. - transfer: The transfer being attempted. - - Returns: - DownloadResult | None: None when the response is usable. - """ - status = response.status_code - if status in (_NOT_FOUND, _GONE): - transfer.part_path.unlink(missing_ok=True) - return DownloadResult( - outcome=Outcome.NOT_FOUND, - bytes_done=0, - message=f"HTTP {status}", - ) - encoding = response.headers.get("Content-Encoding", "identity") - if encoding != "identity": - # We asked for identity. A compressed body means the far side is not - # playing by the rules, and requests would silently expand it -- a - # 200 KB response can decode to 200 MB, and the md5 would still match - # because the same party published it. - return DownloadResult( - outcome=Outcome.TRANSIENT, - bytes_done=size_of(transfer.part_path), - message=f"refused Content-Encoding: {encoding}", - ) - if status == _RANGE_NOT_SATISFIABLE: - # The leftover part is longer than the file actually is; start over. - transfer.part_path.unlink(missing_ok=True) - return DownloadResult( - outcome=Outcome.TRANSIENT, - bytes_done=0, - message="range rejected; restarting", - ) - if status == _TOO_MANY or status >= _SERVER_ERROR: - return DownloadResult( - outcome=Outcome.TRANSIENT, - bytes_done=size_of(transfer.part_path), - retry_after=parse_retry_after(response.headers.get("Retry-After")), - message=f"HTTP {status}", - ) - if status not in (_OK, _PARTIAL): - return DownloadResult( - outcome=Outcome.TRANSIENT, - bytes_done=size_of(transfer.part_path), - message=f"unexpected HTTP {status}", - ) - return None - - -def _over_budget(written: int, expected: int) -> bool: - """Report whether a transfer outgrew the size the API advertised. - - Without this a server that lies about Content-Length can write until the - disk is full; the md5 check alone does not help, because whoever serves the - bytes also publishes the digest. - - Args: - written: Bytes now on disk. - expected: Size the API declared; zero means it did not say. - - Returns: - bool: True when the transfer should be abandoned. - """ - if expected <= 0: - return False - return written > expected + SIZE_SLACK_BYTES - - -def _write_body( - response: requests.Response, - transfer: Transfer, - *, - append: bool, -) -> int | None: - """Write the response body to the part file. - - Args: - response: Streamed response. - transfer: What is being fetched. - append: Whether to continue an existing part file. - - Returns: - int | None: Bytes written, or None when shutdown interrupted it. - """ - mode = "ab" if append else "wb" - done = size_of(transfer.part_path) if append else 0 - with transfer.part_path.open(mode) as handle: - for chunk in response.iter_content(chunk_size=CHUNK_BYTES): - if transfer.should_stop(): - return None - if not chunk: - continue - handle.write(chunk) - done += len(chunk) - if transfer.on_progress is not None: - transfer.on_progress(done) - if _over_budget(done, transfer.expected_size): - return done - return done - - -def size_of(path: Path) -> int: - """Return the size of *path*, or zero when it does not exist. - - Args: - path: File to measure. - - Returns: - int: Size in bytes. - """ - return path.stat().st_size if path.exists() else 0