Make a queued gatelock wait observable while it is happening

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Krzysztof Rudnicki 2026-08-30 15:02:23 +02:00
parent 305e59b3e8
commit 109577e646
3 changed files with 41 additions and 1 deletions

View File

@ -36,6 +36,7 @@ from gatelock._outputs import (
) )
from gatelock._queue import ( from gatelock._queue import (
QUEUE_DEADLINE_SECONDS, QUEUE_DEADLINE_SECONDS,
QUEUE_HEARTBEAT_SECONDS,
QUEUE_POLL_SECONDS, QUEUE_POLL_SECONDS,
QueueResult, QueueResult,
stronger_claims, stronger_claims,
@ -76,6 +77,7 @@ from gatelock.widgets import (
__all__ = [ __all__ = [
"DEFAULT_WRAP", "DEFAULT_WRAP",
"QUEUE_DEADLINE_SECONDS", "QUEUE_DEADLINE_SECONDS",
"QUEUE_HEARTBEAT_SECONDS",
"QUEUE_POLL_SECONDS", "QUEUE_POLL_SECONDS",
"RANK_DIET_GUARD", "RANK_DIET_GUARD",
"RANK_SCREEN_LOCKER", "RANK_SCREEN_LOCKER",

View File

@ -30,6 +30,11 @@ _logger: Final = logging.getLogger(__name__)
QUEUE_POLL_SECONDS: Final = 2.0 QUEUE_POLL_SECONDS: Final = 2.0
QUEUE_DEADLINE_SECONDS: Final = 6 * 60 * 60 QUEUE_DEADLINE_SECONDS: Final = 6 * 60 * 60
# How often to re-state a still-blocked wait at WARNING. A long wait used
# to produce exactly one INFO line at the moment it started and nothing
# else until it ended: on 2026-08-30 screen_locker sat behind wake_alarm
# for 2h58m and `systemctl status` showed a healthy, active, silent unit.
QUEUE_HEARTBEAT_SECONDS: Final = 300.0
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Callable from collections.abc import Callable
@ -75,6 +80,7 @@ def wait_for_turn(
*, *,
poll: float = QUEUE_POLL_SECONDS, poll: float = QUEUE_POLL_SECONDS,
deadline: float = QUEUE_DEADLINE_SECONDS, deadline: float = QUEUE_DEADLINE_SECONDS,
on_state: Callable[[tuple[str, ...], float], None] | None = None,
sleep: Callable[[float], None] | None = None, sleep: Callable[[float], None] | None = None,
now: Callable[[], float] | None = None, now: Callable[[], float] | None = None,
) -> QueueResult: ) -> QueueResult:
@ -86,6 +92,11 @@ def wait_for_turn(
poll: Seconds between checks. Liveness comes from ``flock``, so a poll: Seconds between checks. Liveness comes from ``flock``, so a
SIGKILLed incumbent is noticed on the very next tick. SIGKILLed incumbent is noticed on the very next tick.
deadline: Runaway backstop in seconds. deadline: Runaway backstop in seconds.
on_state: Called with ``(blocked_by, elapsed)`` each time the
blocked set is first seen or changes, and once with an empty
tuple when the wait ends. Lets a caller publish the wait
somewhere observable -- from outside, a blocked process is
indistinguishable from a hung one.
sleep: Injected for tests. sleep: Injected for tests.
now: Injected monotonic clock, for tests. now: Injected monotonic clock, for tests.
@ -97,6 +108,12 @@ def wait_for_turn(
started = clock() started = clock()
seen: list[str] = [] seen: list[str] = []
next_heartbeat = QUEUE_HEARTBEAT_SECONDS
def _publish(blocked: tuple[str, ...], elapsed: float) -> None:
if on_state is not None:
on_state(blocked, elapsed)
while True: while True:
blockers = stronger_claims(arbiter) blockers = stronger_claims(arbiter)
if not blockers: if not blockers:
@ -108,13 +125,16 @@ def wait_for_turn(
waited, waited,
", ".join(seen), ", ".join(seen),
) )
_publish((), waited)
return QueueResult( return QueueResult(
waited_seconds=waited, blocked_by=tuple(seen), timed_out=False waited_seconds=waited, blocked_by=tuple(seen), timed_out=False
) )
newly_seen = False
for claim in blockers: for claim in blockers:
if claim.app not in seen: if claim.app not in seen:
seen.append(claim.app) seen.append(claim.app)
newly_seen = True
_logger.info( _logger.info(
"%s is queued behind %s (rank %d) -- waiting with no window", "%s is queued behind %s (rank %d) -- waiting with no window",
arbiter.claim.app, arbiter.claim.app,
@ -123,6 +143,23 @@ def wait_for_turn(
) )
elapsed = clock() - started elapsed = clock() - started
if newly_seen:
_publish(tuple(seen), elapsed)
# Re-state a long wait periodically. Silence here is what made a
# 2h58m block look identical to a healthy idle unit.
if elapsed >= next_heartbeat:
_logger.warning(
"%s has been queued behind %s for %.0fs with no window -- "
"still waiting (deadline %.0fs)",
arbiter.claim.app,
", ".join(seen),
elapsed,
deadline,
)
_publish(tuple(seen), elapsed)
while next_heartbeat <= elapsed:
next_heartbeat += QUEUE_HEARTBEAT_SECONDS
if elapsed >= deadline: if elapsed >= deadline:
_logger.error( _logger.error(
"%s waited %.0fs behind %s and gave up waiting -- arming " "%s waited %.0fs behind %s and gave up waiting -- arming "
@ -131,6 +168,7 @@ def wait_for_turn(
elapsed, elapsed,
", ".join(seen), ", ".join(seen),
) )
_publish((), elapsed)
return QueueResult( return QueueResult(
waited_seconds=elapsed, blocked_by=tuple(seen), timed_out=True waited_seconds=elapsed, blocked_by=tuple(seen), timed_out=True
) )

View File

@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project] [project]
name = "gatelock" name = "gatelock"
version = "0.7.2" version = "0.7.3"
description = "Shared per-output lock-window + HMAC log-integrity backend for blocking-overlay apps" description = "Shared per-output lock-window + HMAC log-integrity backend for blocking-overlay apps"
requires-python = ">=3.14" requires-python = ">=3.14"
dependencies = [] # pure stdlib — tkinter is bundled with Python dependencies = [] # pure stdlib — tkinter is bundled with Python