testsAndMisc/python_pkg/wsg_grabber/states.py

"""The lifecycle of a single file, as a pure transition function.

``GONE`` is the load-bearing state. A full first pass over /wsg/ takes hours, so
threads fall off page 10 and files 404 between being catalogued and being
fetched. Without a terminal sink those files would be retried on every future
run forever.
"""

from __future__ import annotations

from enum import StrEnum
from typing import Final

from python_pkg.wsg_grabber.constants import MAX_ATTEMPTS

# ``str, Enum`` not ``StrEnum``: mypy targets 3.10, where ``StrEnum`` is absent
# and would collapse to ``Any``, disabling checking on every member below.


class FileState(StrEnum):
    """Where a file is in its lifecycle."""

    NEW = "new"
    DOWNLOADING = "downloading"
    READY = "ready"
    FAILED = "failed"
    CORRUPT = "corrupt"
    KEPT = "kept"
    PASSED = "passed"
    GONE = "gone"


class FileEvent(StrEnum):
    """Something that happened to a file."""

    CLAIMED = "claimed"
    COMPLETED = "completed"
    NOT_FOUND = "not_found"
    DELETED_UPSTREAM = "deleted_upstream"
    TRANSIENT_ERROR = "transient_error"
    CHECKSUM_MISMATCH = "checksum_mismatch"
    KEPT = "kept"
    PASSED = "passed"
    MISSING_LOCALLY = "missing_locally"


TERMINAL: Final[frozenset[FileState]] = frozenset(
    {FileState.KEPT, FileState.PASSED, FileState.GONE},
)

CLAIMABLE: Final[frozenset[FileState]] = frozenset(
    {FileState.NEW, FileState.FAILED, FileState.CORRUPT},
)


def is_terminal(state: FileState) -> bool:
    """Report whether *state* can never change again.

    Args:
        state: State to classify.

    Returns:
        bool: True when no further transition is possible.
    """
    return state in TERMINAL


def next_state(current: FileState, event: FileEvent, attempts: int) -> FileState:
    """Return the state *current* moves to on *event*.

    Args:
        current: The file's present state.
        event: What happened.
        attempts: Download attempts made so far, including the one that just
            failed. Retryable failures give up once this reaches
            ``MAX_ATTEMPTS``.

    Returns:
        FileState: The new state.

    Raises:
        ValueError: If the transition is not legal, which means a caller has a
            bug rather than the board behaving oddly.
    """
    if current in CLAIMABLE and event is FileEvent.CLAIMED:
        return FileState.DOWNLOADING
    if current in CLAIMABLE and event is FileEvent.DELETED_UPSTREAM:
        return FileState.GONE
    if current is FileState.DOWNLOADING:
        return _from_downloading(event, attempts)
    if current is FileState.READY:
        return _from_ready(event)
    msg = f"illegal transition: {current.value} + {event.value}"
    raise ValueError(msg)


def _from_downloading(event: FileEvent, attempts: int) -> FileState:
    """Resolve a transition out of ``DOWNLOADING``.

    Args:
        event: What happened.
        attempts: Download attempts made so far.

    Returns:
        FileState: The new state.

    Raises:
        ValueError: If *event* cannot follow ``DOWNLOADING``.
    """
    if event is FileEvent.COMPLETED:
        return FileState.READY
    if event is FileEvent.NOT_FOUND:
        return FileState.GONE
    exhausted = attempts >= MAX_ATTEMPTS
    if event is FileEvent.TRANSIENT_ERROR:
        return FileState.GONE if exhausted else FileState.FAILED
    if event is FileEvent.CHECKSUM_MISMATCH:
        return FileState.GONE if exhausted else FileState.CORRUPT
    msg = f"illegal transition: downloading + {event.value}"
    raise ValueError(msg)


def _from_ready(event: FileEvent) -> FileState:
    """Resolve a transition out of ``READY``.

    Upstream deletion is deliberately not handled here: once the bytes are on
    disk it no longer matters that the post was removed.

    Args:
        event: What happened.

    Returns:
        FileState: The new state.

    Raises:
        ValueError: If *event* cannot follow ``READY``.
    """
    if event is FileEvent.KEPT:
        return FileState.KEPT
    if event is FileEvent.PASSED:
        return FileState.PASSED
    if event is FileEvent.MISSING_LOCALLY:
        return FileState.GONE
    msg = f"illegal transition: ready + {event.value}"
    raise ValueError(msg)