mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 20:47:03 +01:00
Krzysztof kuhy Rudnicki
89ed9bc95d
New package. Walks every thread on 4chan's /wsg/, downloads each video it has
not seen, and reviews them one at a time: mpv embedded in a Tk window via --wid
with Keep/Pass/Undo directly beneath the picture.
Dedupe is structural rather than a convention. Every post carrying a file also
carries md5 -- base64 MD5 of the whole file -- so it is the sqlite primary key.
An already-seen video costs zero bytes to skip and a byte-identical repost in
another thread collapses onto the existing row. Verified against a live file:
the API's Nz9OEKdMuMZEdYE6eLTKmA== matched base64(md5(bytes)) exactly.
Undo is unbounded and survives quitting: the verdict trail lives in the index,
not in memory, so yesterday's decisions can still be taken back.
No downloaded video is ever deleted. A pass moves the file to trash/ and leaves
it there; files.apply_move refuses to overwrite an existing destination, so the
guarantee holds by construction rather than per caller.
Three bugs only running it could find, each caught from evidence rather than
guesswork:
- the video stayed black: a <Configure> handler set mpv's geometry, fighting
the automatic sizing --wid already does
- playback never advanced: refresh() ran every tick and re-sent loadfile ~7x
a second, restarting the video continuously
- it froze on one video around the 26th while filenames kept advancing: mpv
pushes an event stream down the same socket and nothing drained it, so the
buffer filled and mpv stopped reading commands. Isolated to command #35
without a drain thread; all 60 honoured with one.
A five-lens review then found nine more, all reproduced before fixing: a failed
mpv start stranded the non-daemon worker forever; undo could clobber a file in
incoming/; a NUL-bearing md5 poisoned the index permanently; an oversized int
killed the worker with no event; gzip bombs on both hosts were accepted; the
board endpoints were fetched unconditionally forever; and Session.missing could
corrupt the undo trail.
mpv runs as a subprocess over its JSON IPC socket rather than via python-mpv,
which resolves libmpv at import time and would fail on CI while passing here.
Third-party surface is requests alone, already declared.
277 package tests, 100% branch coverage on all 22 modules, no suppressions.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
104 lines
3.7 KiB
Python
104 lines
3.7 KiB
Python
"""Tests for the pure scheduling and retry helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from python_pkg.wsg_grabber import scanner
|
|
from python_pkg.wsg_grabber.constants import (
|
|
BACKOFF_BASE_S,
|
|
BACKOFF_CAP_S,
|
|
MIN_REQUEST_INTERVAL_S,
|
|
)
|
|
from python_pkg.wsg_grabber.models import ResumePlan, TaskKind, ThreadRef
|
|
|
|
|
|
def test_downloads_are_preferred_so_a_video_appears_early() -> None:
|
|
"""The user watches while the rest load; fetching must outrank scanning."""
|
|
assert scanner.choose_task(5, 10, catalog_fresh=True) is TaskKind.DOWNLOAD
|
|
assert scanner.choose_task(1, 0, catalog_fresh=False) is TaskKind.DOWNLOAD
|
|
|
|
|
|
def test_scanning_happens_when_nothing_is_queued() -> None:
|
|
assert scanner.choose_task(0, 3, catalog_fresh=True) is TaskKind.SCAN
|
|
assert scanner.choose_task(0, 0, catalog_fresh=False) is TaskKind.SCAN
|
|
|
|
|
|
def test_idle_only_when_the_board_is_exhausted() -> None:
|
|
assert scanner.choose_task(0, 0, catalog_fresh=True) is TaskKind.IDLE
|
|
|
|
|
|
def test_stale_threads_picks_unseen_and_bumped_threads() -> None:
|
|
known = {1: 100, 2: 200}
|
|
live = [ThreadRef(1, 100), ThreadRef(2, 250), ThreadRef(3, 10)]
|
|
assert [ref.thread_no for ref in scanner.stale_threads(known, live)] == [2, 3]
|
|
|
|
|
|
def test_stale_threads_is_empty_on_an_unchanged_board() -> None:
|
|
known = {1: 100}
|
|
assert scanner.stale_threads(known, [ThreadRef(1, 100)]) == []
|
|
|
|
|
|
def test_next_delay_enforces_one_request_per_second() -> None:
|
|
assert scanner.next_delay(0.0, 0.0) == pytest.approx(MIN_REQUEST_INTERVAL_S)
|
|
assert scanner.next_delay(10.0, 10.5) == pytest.approx(0.5)
|
|
|
|
|
|
def test_next_delay_never_goes_negative() -> None:
|
|
assert scanner.next_delay(10.0, 99.0) == 0.0
|
|
|
|
|
|
def test_backoff_doubles_and_then_caps() -> None:
|
|
assert scanner.backoff_seconds(1) == pytest.approx(BACKOFF_BASE_S)
|
|
assert scanner.backoff_seconds(2) == pytest.approx(BACKOFF_BASE_S * 2)
|
|
assert scanner.backoff_seconds(3) == pytest.approx(BACKOFF_BASE_S * 4)
|
|
assert scanner.backoff_seconds(99) == pytest.approx(BACKOFF_CAP_S)
|
|
|
|
|
|
def test_backoff_treats_attempt_zero_as_the_first_failure() -> None:
|
|
assert scanner.backoff_seconds(0) == pytest.approx(BACKOFF_BASE_S)
|
|
|
|
|
|
def test_an_explicit_retry_after_wins_but_is_still_capped() -> None:
|
|
assert scanner.backoff_seconds(1, retry_after=7.0) == pytest.approx(7.0)
|
|
assert scanner.backoff_seconds(1, retry_after=9999.0) == pytest.approx(
|
|
BACKOFF_CAP_S,
|
|
)
|
|
|
|
|
|
def test_a_zero_retry_after_falls_back_to_exponential_backoff() -> None:
|
|
assert scanner.backoff_seconds(2, retry_after=0.0) == pytest.approx(
|
|
BACKOFF_BASE_S * 2,
|
|
)
|
|
|
|
|
|
def test_parse_retry_after_handles_the_forms_we_accept() -> None:
|
|
assert scanner.parse_retry_after("30") == pytest.approx(30.0)
|
|
assert scanner.parse_retry_after(" 12 ") == pytest.approx(12.0)
|
|
assert scanner.parse_retry_after(None) is None
|
|
assert scanner.parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT") is None
|
|
assert scanner.parse_retry_after("-5") is None
|
|
|
|
|
|
def test_resume_plan_starts_from_scratch_when_nothing_is_on_disk() -> None:
|
|
plan = scanner.resume_plan(0, 1000)
|
|
assert plan == ResumePlan(offset=0, discard=False)
|
|
|
|
|
|
def test_resume_plan_continues_a_genuine_partial() -> None:
|
|
plan = scanner.resume_plan(400, 1000)
|
|
assert plan.offset == 400
|
|
assert not plan.discard
|
|
|
|
|
|
def test_resume_plan_bins_an_oversized_leftover() -> None:
|
|
"""A .part at or beyond the expected size is corrupt, not resumable."""
|
|
assert scanner.resume_plan(1000, 1000).discard
|
|
assert scanner.resume_plan(1500, 1000).discard
|
|
|
|
|
|
def test_resume_plan_with_unknown_expected_size_still_resumes() -> None:
|
|
plan = scanner.resume_plan(400, 0)
|
|
assert plan.offset == 400
|
|
assert not plan.discard
|