From 109577e6462bfeb319c02eff8a12f6996e739a0c Mon Sep 17 00:00:00 2001
From: Krzysztof kuhy Rudnicki <krzysztofrudnicki0@gmail.com>
Date: Sun, 30 Aug 2026 15:02:23 +0200
Subject: [PATCH] Make a queued gatelock wait observable while it is happening

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 gatelock/gatelock/__init__.py |  2 ++
 gatelock/gatelock/_queue.py   | 38 +++++++++++++++++++++++++++++++++++
 gatelock/pyproject.toml       |  2 +-
 3 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/gatelock/gatelock/__init__.py b/gatelock/gatelock/__init__.py
index bc9db06..299970f 100644
--- a/gatelock/gatelock/__init__.py
+++ b/gatelock/gatelock/__init__.py
@@ -36,6 +36,7 @@ from gatelock._outputs import (
 )
 from gatelock._queue import (
     QUEUE_DEADLINE_SECONDS,
+    QUEUE_HEARTBEAT_SECONDS,
     QUEUE_POLL_SECONDS,
     QueueResult,
     stronger_claims,
@@ -76,6 +77,7 @@ from gatelock.widgets import (
 __all__ = [
     "DEFAULT_WRAP",
     "QUEUE_DEADLINE_SECONDS",
+    "QUEUE_HEARTBEAT_SECONDS",
     "QUEUE_POLL_SECONDS",
     "RANK_DIET_GUARD",
     "RANK_SCREEN_LOCKER",
diff --git a/gatelock/gatelock/_queue.py b/gatelock/gatelock/_queue.py
index aa1cbd8..1c4bf6c 100644
--- a/gatelock/gatelock/_queue.py
+++ b/gatelock/gatelock/_queue.py
@@ -30,6 +30,11 @@ _logger: Final = logging.getLogger(__name__)
 
 QUEUE_POLL_SECONDS: Final = 2.0
 QUEUE_DEADLINE_SECONDS: Final = 6 * 60 * 60
+# How often to re-state a still-blocked wait at WARNING. A long wait used
+# to produce exactly one INFO line at the moment it started and nothing
+# else until it ended: on 2026-08-30 screen_locker sat behind wake_alarm
+# for 2h58m and `systemctl status` showed a healthy, active, silent unit.
+QUEUE_HEARTBEAT_SECONDS: Final = 300.0
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -75,6 +80,7 @@ def wait_for_turn(
     *,
     poll: float = QUEUE_POLL_SECONDS,
     deadline: float = QUEUE_DEADLINE_SECONDS,
+    on_state: Callable[[tuple[str, ...], float], None] | None = None,
     sleep: Callable[[float], None] | None = None,
     now: Callable[[], float] | None = None,
 ) -> QueueResult:
@@ -86,6 +92,11 @@ def wait_for_turn(
         poll: Seconds between checks. Liveness comes from ``flock``, so a
             SIGKILLed incumbent is noticed on the very next tick.
         deadline: Runaway backstop in seconds.
+        on_state: Called with ``(blocked_by, elapsed)`` each time the
+            blocked set is first seen or changes, and once with an empty
+            tuple when the wait ends. Lets a caller publish the wait
+            somewhere observable -- from outside, a blocked process is
+            indistinguishable from a hung one.
         sleep: Injected for tests.
         now: Injected monotonic clock, for tests.
 
@@ -97,6 +108,12 @@ def wait_for_turn(
 
     started = clock()
     seen: list[str] = []
+    next_heartbeat = QUEUE_HEARTBEAT_SECONDS
+
+    def _publish(blocked: tuple[str, ...], elapsed: float) -> None:
+        if on_state is not None:
+            on_state(blocked, elapsed)
+
     while True:
         blockers = stronger_claims(arbiter)
         if not blockers:
@@ -108,13 +125,16 @@ def wait_for_turn(
                     waited,
                     ", ".join(seen),
                 )
+            _publish((), waited)
             return QueueResult(
                 waited_seconds=waited, blocked_by=tuple(seen), timed_out=False
             )
 
+        newly_seen = False
         for claim in blockers:
             if claim.app not in seen:
                 seen.append(claim.app)
+                newly_seen = True
                 _logger.info(
                     "%s is queued behind %s (rank %d) -- waiting with no window",
                     arbiter.claim.app,
@@ -123,6 +143,23 @@ def wait_for_turn(
                 )
 
         elapsed = clock() - started
+        if newly_seen:
+            _publish(tuple(seen), elapsed)
+        # Re-state a long wait periodically. Silence here is what made a
+        # 2h58m block look identical to a healthy idle unit.
+        if elapsed >= next_heartbeat:
+            _logger.warning(
+                "%s has been queued behind %s for %.0fs with no window -- "
+                "still waiting (deadline %.0fs)",
+                arbiter.claim.app,
+                ", ".join(seen),
+                elapsed,
+                deadline,
+            )
+            _publish(tuple(seen), elapsed)
+            while next_heartbeat <= elapsed:
+                next_heartbeat += QUEUE_HEARTBEAT_SECONDS
+
         if elapsed >= deadline:
             _logger.error(
                 "%s waited %.0fs behind %s and gave up waiting -- arming "
@@ -131,6 +168,7 @@ def wait_for_turn(
                 elapsed,
                 ", ".join(seen),
             )
+            _publish((), elapsed)
             return QueueResult(
                 waited_seconds=elapsed, blocked_by=tuple(seen), timed_out=True
             )
diff --git a/gatelock/pyproject.toml b/gatelock/pyproject.toml
index bcbc040..6507c63 100644
--- a/gatelock/pyproject.toml
+++ b/gatelock/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gatelock"
-version = "0.7.2"
+version = "0.7.3"
 description = "Shared per-output lock-window + HMAC log-integrity backend for blocking-overlay apps"
 requires-python = ">=3.14"
 dependencies = []  # pure stdlib — tkinter is bundled with Python