testsAndMisc/python_pkg/wsg_grabber/downloader.py

"""The background worker: walk the board, fetch files, announce each one.

Structured so that :meth:`Worker.run_once` performs exactly one unit of work
and returns. The threaded loop is a thin wrapper around it, which is what lets
the tests drive the whole state machine synchronously without starting a thread
-- important because pytest treats warnings as errors and a leaked thread would
fail the suite.

The worker never touches Tk. It publishes events onto a queue that the UI
drains from its own event loop.
"""

from __future__ import annotations

from dataclasses import dataclass, field
import logging
import sqlite3
import threading
import time
from typing import TYPE_CHECKING

import requests

from python_pkg.wsg_grabber import net, scanner, store
from python_pkg.wsg_grabber._fetch import DownloadMixin
from python_pkg.wsg_grabber._scan import ScanMixin
from python_pkg.wsg_grabber.constants import (
    IDLE_POLL_S,
    IDLE_RESCAN_S,
)
from python_pkg.wsg_grabber.models import (
    JsonResponse,
    ScanFinished,
    TaskKind,
    ThreadRef,
    WorkerFailed,
)

_LOGGER = logging.getLogger(__name__)

# Every failure family this worker can plausibly hit: the network, the disk,
# the index, and bad data coming back from the board. Named explicitly rather
# than catching Exception, which the linters reject and which would also
# swallow genuine programming errors that ought to be loud.
_RECOVERABLE = (
    ArithmeticError,
    OSError,
    RuntimeError,
    ValueError,
    LookupError,
    TypeError,
    AttributeError,
    sqlite3.Error,
    requests.RequestException,
)

if TYPE_CHECKING:
    from collections.abc import Callable
    from pathlib import Path
    import queue

    from python_pkg.wsg_grabber.models import DownloadEvent


@dataclass
class WorkerDeps:
    """Collaborators supplied to the worker, so tests can swap them out."""

    conn: sqlite3.Connection
    session: requests.Session
    incoming: Path
    events: queue.SimpleQueue[DownloadEvent]
    stop: threading.Event = field(default_factory=threading.Event)
    include_archive: bool = True


class Worker(ScanMixin, DownloadMixin):
    """Walks /wsg/ and downloads every file it has not seen before.

    The board-walking and file-fetching halves live in _scan and _fetch to
    keep this module under the 250-line cap; they are mixins because the
    tests drive their methods as bound attributes of a Worker.
    """

    def __init__(self, deps: WorkerDeps) -> None:
        """Store collaborators and initialise scan bookkeeping.

        Args:
            deps: Connection, session, destination and event queue.
        """
        self._deps = deps
        self._pending_threads: list[ThreadRef] = []
        self._catalog_fresh = False
        self._downloaded = 0
        self._last_request = 0.0
        self._finished_announced = False
        self._sweep_finished_at = 0.0

    @property
    def downloaded(self) -> int:
        """Return how many files this worker has fetched.

        Returns:
            int: Completed downloads since construction.
        """
        return self._downloaded

    def run(self) -> None:
        """Loop until the stop flag is set, reporting any crash to the UI.

        Failures on a background thread would otherwise vanish, leaving the
        reviewer waiting for downloads that never arrive, so anything in
        ``_RECOVERABLE`` is logged and surfaced in the status bar. The session
        and connection are closed on every exit path. The suite does not police
        that for us: ``meta/pyproject.toml`` downgrades
        ``PytestUnraisableExceptionWarning``, so a leak here would pass CI
        unnoticed.
        """
        try:
            while not self._deps.stop.is_set():
                self.run_once()
        except _RECOVERABLE as exc:
            _LOGGER.exception("wsg download worker stopped")
            self._publish(WorkerFailed(message=str(exc)))
        finally:
            self._deps.session.close()
            self._deps.conn.close()

    def run_once(self) -> TaskKind:
        """Perform one unit of work and return what it was.

        Returns:
            TaskKind: The action taken, so a caller can drive the worker step
            by step.
        """
        task = scanner.choose_task(
            store.pending_downloads(self._deps.conn),
            len(self._pending_threads),
            catalog_fresh=self._catalog_fresh,
        )
        if task is TaskKind.DOWNLOAD:
            self._download_one()
        elif task is TaskKind.SCAN:
            self._scan_step()
        else:
            self._idle()
        return task

    def _idle(self) -> None:
        """Report the board exhausted, then poll again after a quiet period.

        The rescan is deliberately not immediate. Resetting the catalogue every
        idle tick would walk the board once a second and re-announce completion
        each time; waiting keeps a finished sweep quiet until there is a real
        chance of new content.
        """
        self._announce_finished()
        self._deps.stop.wait(IDLE_POLL_S)
        if time.monotonic() - self._sweep_finished_at >= IDLE_RESCAN_S:
            self._catalog_fresh = False
            self._finished_announced = False

    def _announce_finished(self) -> None:
        """Emit ScanFinished once per completed sweep of the board."""
        if self._finished_announced:
            return
        self._finished_announced = True
        self._sweep_finished_at = time.monotonic()
        self._publish(ScanFinished())

    def _get_json(self, url: str, stamp: str | None = None) -> JsonResponse:
        """Fetch JSON while respecting the one-request-per-second rule.

        Args:
            url: Absolute API URL.
            stamp: Stored ``Last-Modified`` to send conditionally.

        Returns:
            JsonResponse: The decoded response.
        """
        self._wait_for_slot()
        return net.get_json(self._deps.session, url, stamp)

    def _wait_for_slot(self) -> None:
        """Sleep just long enough to stay within the API's rate limit."""
        now = time.monotonic()
        delay = scanner.next_delay(self._last_request, now)
        if delay > 0:
            self._deps.stop.wait(delay)
        self._last_request = time.monotonic()

    def _publish(self, event: DownloadEvent) -> None:
        """Hand an event to the UI thread.

        Args:
            event: What happened.
        """
        self._deps.events.put(event)


def start(
    build_deps: Callable[[], WorkerDeps],
    on_failure: Callable[[str], None] = lambda _message: None,
) -> threading.Thread:
    """Run a worker on its own non-daemon thread.

    A *factory* rather than ready-made dependencies, because sqlite forbids
    using a connection from any thread but the one that opened it. Building
    them inside the thread keeps ownership and disposal on the same side of the
    boundary; passing a pre-opened connection in raises ProgrammingError the
    moment the worker closes it.

    Args:
        build_deps: Called on the new thread to open the connection, session
            and everything else the worker owns.
        on_failure: Called with a message when *build_deps* itself fails, so
            the UI learns there will be no downloads.

    Returns:
        threading.Thread: The started thread; set the stop flag, then join it.
    """

    def _run() -> None:
        try:
            deps = build_deps()
        except _RECOVERABLE as exc:
            # Worker.run reports its own failures, but a crash while building
            # the connection and session happens before there is a Worker at
            # all. Without this the reviewer waits on downloads forever and the
            # only trace is threading.excepthook.
            _LOGGER.exception("wsg worker could not start")
            on_failure(str(exc))
            return
        Worker(deps).run()

    thread = threading.Thread(target=_run, name="wsg-downloader", daemon=False)
    thread.start()
    return thread