mirror of
https://github.com/kuhyx/utils.git
synced 2026-11-10 20:27:15 +01:00
Coach relay (Python side): loopback API + RTDB phone path for the guided coaching UI
leetcode_coach.relay implements DOCS-coach-relay.md: one paced serial lane (1 start / 3 s, 5 queued, done-map re-answers a rid), the loopback API on 127.0.0.1:8777 (Host check, JSON only, 16 KiB), the RTDB request/result/ state/chat/heartbeat mirror over SSE with 60 s rescans and 24 h cleanup, the five-step state machine derived from the session log, graders (steps 1/5 haiku -> Ollama, fail open "unverified"; step 2 through PhaseGate.screen; step 3 mechanical against answer_key.json), the CLI's sonnet chat with the log in the child env and the local hand-over, run-code on node --permission, and step 4 through the harvester's read-only fetch_recent_ac once a minute (three-valued: unknown is never "not yet"). New log events session_start/grade/chat/session_end land in session_log, _coach_log and _well_formed together; the snapshot rows gain grades and accepted (_coach_grades). A forced step-2 grade opens code (phase.req_state). leetcode-coach-relay.service + an install.sh step (not run). Contract fixes in the same commit: step 3 counts also_accepted and combined entries' parts, step-2 grade events and the force event shape, next_step, per-call model caps instead of a 280 s watchdog, heartbeat on busy changes, the statement fetched in-process, enable + restart on reinstall. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PjvF5DZG9JduSBYKxqyysi
This commit is contained in:
parent
2a9732a312
commit
f47ed80584
@ -37,17 +37,26 @@ def claude_bin() -> str:
|
||||
|
||||
|
||||
def ask(
|
||||
args: list[str], prompt: str, *, timeout: float | None = None
|
||||
args: list[str],
|
||||
prompt: str,
|
||||
*,
|
||||
timeout: float | None = None,
|
||||
env: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""One `claude -p` turn; the JSON result, or a stand-in carrying stderr."""
|
||||
env = {**os.environ, "CLAUDE_TTY_FIX_OFF": "1"}
|
||||
"""One `claude -p` turn; the JSON result, or a stand-in carrying stderr.
|
||||
|
||||
`env` is merged over this process's environment for the child only: the
|
||||
relay hands each session its own ``LEETCODE_COACH_LOG`` this way, never
|
||||
through ``os.environ`` (one process serves many sessions).
|
||||
"""
|
||||
child_env = {**os.environ, "CLAUDE_TTY_FIX_OFF": "1", **(env or {})}
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[claude_bin(), "-p", *args, "--output-format", "json", prompt],
|
||||
capture_output=True,
|
||||
stdin=subprocess.DEVNULL, # claude -p would otherwise drain REPL input
|
||||
text=True,
|
||||
env=env,
|
||||
env=child_env,
|
||||
check=False,
|
||||
cwd=Path.home(),
|
||||
timeout=timeout,
|
||||
|
||||
@ -87,9 +87,13 @@ def chat(
|
||||
*,
|
||||
post: Callable[[str, dict[str, Any] | None, float], dict[str, Any]] = http_json,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
timeout: float = REQUEST_TIMEOUT_S,
|
||||
) -> LocalReply:
|
||||
"""Ask the local model for the next assistant message.
|
||||
|
||||
`timeout` bounds the HTTP call; the relay passes less than the default so
|
||||
a whole request fits its cap (DOCS-coach-relay.md section 1).
|
||||
|
||||
Raises:
|
||||
ContextOverflowError: when the prompt would not fit in `NUM_CTX`.
|
||||
"""
|
||||
@ -105,7 +109,7 @@ def chat(
|
||||
"options": {"num_ctx": NUM_CTX, "temperature": TEMPERATURE},
|
||||
}
|
||||
start = clock()
|
||||
data = post(f"{OLLAMA_URL}/api/chat", payload, REQUEST_TIMEOUT_S)
|
||||
data = post(f"{OLLAMA_URL}/api/chat", payload, timeout)
|
||||
seconds = clock() - start
|
||||
prompt_tokens = int(data.get("prompt_eval_count") or 0)
|
||||
if prompt_tokens >= NUM_CTX:
|
||||
|
||||
@ -123,6 +123,30 @@ def build_apkg(slug: str, cards: list[card_rules.Card], today: str) -> Path:
|
||||
return out
|
||||
|
||||
|
||||
def deliver(
|
||||
slug: str,
|
||||
cards: list[card_rules.Card],
|
||||
*,
|
||||
replace: bool = False,
|
||||
online: bool | None = None,
|
||||
) -> str:
|
||||
"""Log already-validated `cards` and deliver them; the delivery line.
|
||||
|
||||
AnkiConnect when Anki is open, else an .apkg. The coach relay calls this
|
||||
in-process (no ``postmortem`` here: the caller logs it to its own log).
|
||||
"""
|
||||
today = dt.datetime.now(tz=dt.UTC).date().isoformat()
|
||||
_log(slug, cards, today)
|
||||
if anki_connect.available() if online is None else online:
|
||||
done = push_direct(slug, cards, replace=replace)
|
||||
return f"{len(cards)} cards -> Anki desktop: {done}"
|
||||
out = build_apkg(slug, cards, today)
|
||||
return (
|
||||
f"{len(cards)} cards -> {out} (Anki desktop not open). Import with:\n"
|
||||
f"anki {shlex.quote(str(out))}"
|
||||
)
|
||||
|
||||
|
||||
def _usage() -> str:
|
||||
"""The `Usage:` line of this module's docstring."""
|
||||
return (__doc__ or "").strip().splitlines()[2]
|
||||
@ -153,16 +177,8 @@ def main() -> int:
|
||||
"--replace needs Anki open (AnkiConnect); nothing changed", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
today = dt.datetime.now(tz=dt.UTC).date().isoformat()
|
||||
_log(slug, cards, today)
|
||||
print(card_rules.table(cards))
|
||||
if online:
|
||||
done = push_direct(slug, cards, replace=replace)
|
||||
print(f"{len(cards)} cards -> Anki desktop: {done}")
|
||||
else:
|
||||
out = build_apkg(slug, cards, today)
|
||||
print(f"{len(cards)} cards -> {out} (Anki desktop not open). Import with:")
|
||||
print(f"anki {shlex.quote(str(out))}")
|
||||
print(deliver(slug, cards, replace=replace, online=online))
|
||||
# Delivered: the post-mortem is done. Re-test completion and "solved"
|
||||
# (progress.py) read this event from the REPL's session log.
|
||||
session_log.append_current("postmortem", cards=len(cards))
|
||||
|
||||
@ -73,15 +73,42 @@ class ReqState:
|
||||
return self.accepted or self.forced
|
||||
|
||||
|
||||
REQ_STEP = 2
|
||||
"""The app's step 2 (requirements) in ``grade`` events."""
|
||||
|
||||
|
||||
def _step2_grades(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
e for e in events if e.get("event") == "grade" and e.get("step") == REQ_STEP
|
||||
]
|
||||
|
||||
|
||||
def req_state(events: list[dict[str, Any]]) -> ReqState:
|
||||
"""Accepted on a pass/unverified check; forced after `MAX_LISTS` incompletes."""
|
||||
"""Accepted on a pass/unverified check; forced after `MAX_LISTS` incompletes.
|
||||
|
||||
A step-2 ``grade`` with ``forced: true`` (the app's "continue anyway")
|
||||
forces it too, or a forced step 2 would still refuse kuhy's code.
|
||||
"""
|
||||
verdicts = [e.get("verdict") for e in events if e.get("event") == "req_check"]
|
||||
tries = verdicts.count("incomplete")
|
||||
accepted = "pass" in verdicts or "unverified" in verdicts
|
||||
return ReqState(accepted, not accepted and tries >= MAX_LISTS, tries)
|
||||
pushed = any(e.get("forced") is True for e in _step2_grades(events))
|
||||
forced = not accepted and (tries >= MAX_LISTS or pushed)
|
||||
return ReqState(accepted, forced, tries)
|
||||
|
||||
|
||||
def _property_line(prop: str, box: toolbox.Toolbox) -> str:
|
||||
def step2_attempts(events: list[dict[str, Any]]) -> int:
|
||||
"""Failed step-2 tries: incomplete lists plus app ``fail`` grades.
|
||||
|
||||
A ``fail`` (the list read as code, or named an operation) never reaches
|
||||
the checker, so it has no ``req_check``; a ``partial`` does, and is
|
||||
already counted there. CLI and app count one file alike.
|
||||
"""
|
||||
fails = sum(1 for e in _step2_grades(events) if e.get("grade") == "fail")
|
||||
return req_state(events).tries + fails
|
||||
|
||||
|
||||
def property_line(prop: str, box: toolbox.Toolbox) -> str:
|
||||
"""`prop: meaning`, or just `prop` when the meaning names an entry.
|
||||
|
||||
Some meanings illustrate with the operation itself (self-inverse:
|
||||
@ -99,7 +126,7 @@ def _req_note(result: req_check.Check, state: ReqState, box: toolbox.Toolbox) ->
|
||||
if state.forced:
|
||||
missing = ", ".join(result.missing)
|
||||
return f"[req list still incomplete ({missing}) -- code accepted anyway]"
|
||||
props = "\n".join(f" - {_property_line(p, box)}" for p in result.missing)
|
||||
props = "\n".join(f" - {property_line(p, box)}" for p in result.missing)
|
||||
return (
|
||||
f"[req list incomplete -- it does not yet say these properties:\n{props}\n"
|
||||
f" revise with req: ({state.tries} of {MAX_LISTS})]"
|
||||
@ -133,24 +160,34 @@ class PhaseGate:
|
||||
self.known = self.source()
|
||||
return self.known
|
||||
|
||||
def events(self) -> list[dict[str, Any]]:
|
||||
"""This session's log events (`session_log.current`)."""
|
||||
return session_log.current(session_log.read(self.log))
|
||||
|
||||
def code_seen(self) -> bool:
|
||||
"""Whether a code block has been let through this session."""
|
||||
return session_log.has_code(self.log)
|
||||
|
||||
def screen(self, block: str) -> tuple[str | None, str]:
|
||||
"""(what to send the coach, or None to refuse; a note for kuhy)."""
|
||||
def screen(
|
||||
self, block: str, *, is_code: bool | None = None
|
||||
) -> tuple[str | None, str]:
|
||||
"""(what to send the coach, or None to refuse; a note for kuhy).
|
||||
|
||||
`is_code` is a verdict the caller already reached (the relay judges a
|
||||
message before its own step-2 check), so the model is not asked twice.
|
||||
"""
|
||||
text = block.strip()
|
||||
if text.lower() == "reveal":
|
||||
session_log.append(self.log, "reveal")
|
||||
return block, ""
|
||||
if REQ_RE.match(text):
|
||||
return self._req(block)
|
||||
verdict = regex_verdict(text)
|
||||
verdict = regex_verdict(text) if is_code is None else is_code
|
||||
if verdict is None:
|
||||
verdict = req_check.looks_like_code(text, self.ask)
|
||||
if not verdict:
|
||||
return block, ""
|
||||
state = req_state(session_log.read(self.log))
|
||||
state = req_state(self.events())
|
||||
if not state.open:
|
||||
return None, self._refusal(state)
|
||||
extra = {"req_incomplete": True} if state.forced else {}
|
||||
@ -167,12 +204,16 @@ class PhaseGate:
|
||||
missing=list(result.missing),
|
||||
via=result.via,
|
||||
)
|
||||
state = req_state(session_log.read(self.log))
|
||||
state = req_state(self.events())
|
||||
tag = result.verdict
|
||||
if result.missing:
|
||||
tag += "; missing properties: " + ", ".join(result.missing)
|
||||
return f"{block}\n\n[req check: {tag}]", _req_note(result, state, self.toolbox)
|
||||
|
||||
def refusal_note(self) -> str:
|
||||
"""The line a refused code block gets now (the relay's ``note``)."""
|
||||
return self._refusal(req_state(self.events()))
|
||||
|
||||
@staticmethod
|
||||
def _refusal(state: ReqState) -> str:
|
||||
if not state.tries:
|
||||
|
||||
11
leetcode_coach/leetcode_coach/relay/__init__.py
Normal file
11
leetcode_coach/leetcode_coach/relay/__init__.py
Normal file
@ -0,0 +1,11 @@
|
||||
"""The coach relay: one PC service behind the app's guided coaching UI.
|
||||
|
||||
``DOCS-coach-relay.md`` is the contract. The desktop app reaches it over a
|
||||
loopback HTTP API (``http_api``), the phone through Firebase RTDB (``phone``);
|
||||
both feed one paced serial lane (``lane``) into the session backend
|
||||
(``coach``), which grades the five steps and runs the coaching chat.
|
||||
|
||||
Nothing here can reach the gate's ledger, credits or unlocks: the only
|
||||
``leetcode_guard`` modules in the import closure are the read-only
|
||||
submission probe and its client (``test_relay_boundary.py``).
|
||||
"""
|
||||
64
leetcode_coach/leetcode_coach/relay/__main__.py
Normal file
64
leetcode_coach/leetcode_coach/relay/__main__.py
Normal file
@ -0,0 +1,64 @@
|
||||
"""``python3 -m leetcode_coach.relay``: the systemd unit's entry point.
|
||||
|
||||
Logs to stderr (the journal) at INFO; SIGTERM stops taking requests and
|
||||
lets the running one finish (the unit's ``KillMode=mixed`` spares the claude
|
||||
child meanwhile). The phone path signs in as crdt_sync app
|
||||
``leetcode_coach_relay`` via `_snapshot_auth.connect`, which heals from a
|
||||
sibling app's session; ``$LEETCODE_COACH_RTDB_URL`` skips sign-in (tests).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from crdt_sync import CONFIG_FILE, FirebaseConfig
|
||||
|
||||
from leetcode_coach.relay import config
|
||||
from leetcode_coach.relay.app import Relay
|
||||
from leetcode_coach.relay.coach import Deps
|
||||
from leetcode_coach.relay.rtdb import Rtdb
|
||||
from leetcode_guard import _snapshot_auth
|
||||
|
||||
FAKE_ID = "relay-test"
|
||||
"""What the relay hands the fake RTDB, which takes any non-empty id token."""
|
||||
|
||||
|
||||
def make_rtdb() -> Rtdb:
|
||||
"""The signed-in RTDB client (or the fake one the env var names)."""
|
||||
url = config.rtdb_url()
|
||||
if url:
|
||||
return Rtdb(url, lambda: FAKE_ID)
|
||||
client = _snapshot_auth.connect(config.APP_NAME, timeout_seconds=15)
|
||||
database = FirebaseConfig.load(CONFIG_FILE).database_url
|
||||
return Rtdb(database, client.auth.id_token)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run until SIGTERM / SIGINT."""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
stream=sys.stderr,
|
||||
format="%(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
relay = Relay(Deps())
|
||||
|
||||
def stop(_signum: int, _frame: object) -> None:
|
||||
relay.stop_event.set()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
relay.start()
|
||||
logging.getLogger(__name__).info("coach relay on 127.0.0.1:%d", relay.port)
|
||||
threading.Thread(
|
||||
target=relay.connect_phone, args=(make_rtdb,), name="rtdb", daemon=True
|
||||
).start()
|
||||
relay.stop_event.wait()
|
||||
relay.stop()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
141
leetcode_coach/leetcode_coach/relay/app.py
Normal file
141
leetcode_coach/leetcode_coach/relay/app.py
Normal file
@ -0,0 +1,141 @@
|
||||
"""The relay assembled: backend, lane, loopback API, phone link, ticker.
|
||||
|
||||
One process (``leetcode-coach-relay.service``). Every wait is injectable so
|
||||
the end-to-end tests run the real threads at test speed: the 3 s pace, the
|
||||
Accepted ticker, the heartbeat and the rescan. The phone link connects in
|
||||
its own thread and retries with backoff, so a Firebase outage at boot never
|
||||
delays the desktop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach.relay import config
|
||||
from leetcode_coach.relay.coach import Coach, Deps
|
||||
from leetcode_coach.relay.done import DoneMap
|
||||
from leetcode_coach.relay.http_api import Server
|
||||
from leetcode_coach.relay.lane import PACE_S, Lane
|
||||
from leetcode_coach.relay.phone import HEARTBEAT_S, RESCAN_S, PhoneLink
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from leetcode_coach.relay.rtdb import Rtdb
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
TICK_S: Final = 5.0
|
||||
CONNECT_BACKOFF_S: Final = (5.0, 30.0, 120.0, 300.0)
|
||||
|
||||
|
||||
class Relay: # pylint: disable=too-many-instance-attributes
|
||||
"""All the parts, and the threads that run them."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
deps: Deps,
|
||||
*,
|
||||
port: int | None = None,
|
||||
pace_s: float = PACE_S,
|
||||
tick_s: float = TICK_S,
|
||||
heartbeat_s: float = HEARTBEAT_S,
|
||||
rescan_s: float = RESCAN_S,
|
||||
) -> None:
|
||||
"""Build everything and bind the port; nothing runs until `start`."""
|
||||
self.deps = deps
|
||||
self.started = deps.clock_ms()
|
||||
self.coach = Coach(deps)
|
||||
done = DoneMap(deps.state_dir / "requests_done.jsonl", deps.clock_ms)
|
||||
self.lane = Lane(self.coach.handle, done, clock_ms=deps.clock_ms, pace_s=pace_s)
|
||||
bind = config.port() if port is None else port
|
||||
self.server = Server((config.HOST, bind), self)
|
||||
self.stop_event = threading.Event()
|
||||
self.phone: PhoneLink | None = None
|
||||
self._tick_s = tick_s
|
||||
self._beat_s = heartbeat_s
|
||||
self._rescan_s = rescan_s
|
||||
self._threads: list[threading.Thread] = []
|
||||
|
||||
@property
|
||||
def port(self) -> int:
|
||||
"""The bound loopback port (tests bind port 0)."""
|
||||
return int(self.server.server_address[1])
|
||||
|
||||
def heartbeat(self) -> dict[str, Any]:
|
||||
"""``{"beat", "started", "state_rev", "busy"?, "version"}`` (beat in ms)."""
|
||||
beat: dict[str, Any] = {
|
||||
"beat": self.deps.clock_ms(),
|
||||
"started": self.started,
|
||||
"state_rev": self.coach.rev,
|
||||
"version": 1,
|
||||
}
|
||||
if self.lane.busy is not None:
|
||||
beat["busy"] = self.lane.busy
|
||||
return beat
|
||||
|
||||
def _spawn(self, name: str, target: Callable[[], object]) -> None:
|
||||
thread = threading.Thread(target=target, name=name, daemon=True)
|
||||
thread.start()
|
||||
self._threads.append(thread)
|
||||
|
||||
def _tick_loop(self) -> None:
|
||||
while not self.stop_event.wait(self._tick_s):
|
||||
self.coach.tick()
|
||||
|
||||
def start(self) -> None:
|
||||
"""Serve the API, run the lane and the Accepted ticker."""
|
||||
self._spawn("http", self.server.serve_forever)
|
||||
self._spawn("lane", lambda: self.lane.run(self.stop_event))
|
||||
self._spawn("tick", self._tick_loop)
|
||||
|
||||
def link_phone(self, rtdb: Rtdb) -> PhoneLink:
|
||||
"""Mirror into `rtdb` and take the phone's requests from it."""
|
||||
phone = PhoneLink(
|
||||
rtdb,
|
||||
self.lane,
|
||||
self.coach,
|
||||
self.heartbeat,
|
||||
clock_ms=self.deps.clock_ms,
|
||||
heartbeat_s=self._beat_s,
|
||||
rescan_s=self._rescan_s,
|
||||
)
|
||||
phone.attach()
|
||||
self.phone = phone
|
||||
self._spawn("rtdb-listen", lambda: phone.listen(self.stop_event))
|
||||
self._spawn(
|
||||
"rtdb-tend", lambda: phone.tend(self.stop_event, self.stop_event.wait)
|
||||
)
|
||||
return phone
|
||||
|
||||
def connect_phone(self, make: Callable[[], Rtdb]) -> None:
|
||||
"""Keep trying `make` (Firebase sign-in) with backoff, then link."""
|
||||
failures = 0
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
rtdb = make()
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# Sign-in fails a dozen ways (config, auth, network); each is
|
||||
# an outage to wait out, never a reason to stop the relay.
|
||||
delay = CONNECT_BACKOFF_S[min(failures, len(CONNECT_BACKOFF_S) - 1)]
|
||||
failures += 1
|
||||
_logger.exception(
|
||||
"RTDB not reachable; phone path retries in %.0fs", delay
|
||||
)
|
||||
self.stop_event.wait(delay)
|
||||
continue
|
||||
self.link_phone(rtdb)
|
||||
return
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop taking requests, let the running one finish, close the port."""
|
||||
self.stop_event.set()
|
||||
self.lane.stop()
|
||||
if any(t.name == "http" for t in self._threads):
|
||||
self.server.shutdown() # blocks unless serve_forever runs
|
||||
self.server.server_close()
|
||||
for thread in self._threads:
|
||||
if thread.name == "lane":
|
||||
thread.join(timeout=300)
|
||||
48
leetcode_coach/leetcode_coach/relay/cards.py
Normal file
48
leetcode_coach/leetcode_coach/relay/cards.py
Normal file
@ -0,0 +1,48 @@
|
||||
"""Step 5's reward: the post-mortem's property cards, from the answer key.
|
||||
|
||||
The existing builder, in-process: property -> operation cards for the key's
|
||||
toolbox entries (`toolbox_cards.cards_for`), checked by `card_rules` and
|
||||
delivered by `make_cards.deliver` (AnkiConnect, else an .apkg). The relay
|
||||
appends the ``postmortem`` event itself: ``make_cards.main`` finds the log
|
||||
through ``os.environ``, which one process serving many sessions never sets.
|
||||
A card failure only fills ``error``; the step stays passed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach import card_rules, make_cards, toolbox_cards
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from leetcode_coach.relay.session import Session
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build(
|
||||
session: Session, entries: tuple[str, ...], *, dry_run: bool = False
|
||||
) -> dict[str, Any]:
|
||||
"""Build and deliver the cards; ``{"built", "deck", "error"}``."""
|
||||
out: dict[str, Any] = {"built": 0, "deck": make_cards.DECK_NAME, "error": None}
|
||||
try:
|
||||
cards = toolbox_cards.cards_for(list(entries))
|
||||
except (KeyError, OSError, ValueError) as exc:
|
||||
_logger.warning("no cards for %s: %s", session.slug, exc)
|
||||
return {**out, "error": f"no cards: {exc}"}
|
||||
if errors := card_rules.validate(toolbox_cards.SLUG, cards):
|
||||
_logger.warning("cards rejected for %s: %s", session.slug, errors)
|
||||
return {**out, "error": "cards rejected: " + "; ".join(errors)}
|
||||
if dry_run:
|
||||
return {**out, "built": len(cards)}
|
||||
try:
|
||||
line = make_cards.deliver(toolbox_cards.SLUG, cards)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
# AnkiConnect, genanki and the disk all fail in their own ways; none
|
||||
# of them may cost kuhy the passed step.
|
||||
_logger.exception("card delivery failed for %s", session.slug)
|
||||
return {**out, "error": f"delivery failed: {exc}"}
|
||||
_logger.info("cards for %s: %s", session.slug, line)
|
||||
session.log_event("postmortem", cards=len(cards))
|
||||
return {**out, "built": len(cards)}
|
||||
127
leetcode_coach/leetcode_coach/relay/chat.py
Normal file
127
leetcode_coach/leetcode_coach/relay/chat.py
Normal file
@ -0,0 +1,127 @@
|
||||
"""The coaching chat: the CLI's sonnet session, one per problem.
|
||||
|
||||
The same session `leetcode_repl.run_claude` runs: it opens with
|
||||
``/leetcode <slug>`` (which loads SKILL.md and arms the no-solution Stop
|
||||
gate), then every turn is ``claude -p --resume``. Each call carries this
|
||||
session's ``LEETCODE_COACH_LOG`` in the child's env only, so the Stop hook
|
||||
reads the right log while one relay process serves every session.
|
||||
|
||||
On a usage limit the session continues on the local model with the history
|
||||
so far, as the REPL hands over, and stays there. When that is down too the
|
||||
turn fails ``claude_unavailable`` with the limit's reason and reset time.
|
||||
|
||||
Replies come back RAW: the caller strips and logs the rung marker
|
||||
(`PhaseGate.after_reply`) after it has logged the turn's own events.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
import urllib.error
|
||||
|
||||
from leetcode_coach import claude_cli, local_backend, local_cards, local_coach
|
||||
from leetcode_coach.relay.errors import CoachError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from leetcode_coach.relay.session import Session
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
TURN_TIMEOUT_S: Final = 240.0
|
||||
RESETS_RE: Final = re.compile(r"resets\s+([^\n]+?)\s*$", re.IGNORECASE | re.MULTILINE)
|
||||
TurnError = CoachError
|
||||
"""A coach turn produced no reply (``model_error`` / ``claude_unavailable``)."""
|
||||
|
||||
|
||||
def model() -> str:
|
||||
"""The coaching model (``LEETCODE_MODEL``, default sonnet, as the REPL)."""
|
||||
return os.environ.get("LEETCODE_MODEL") or "sonnet"
|
||||
|
||||
|
||||
def _env(session: Session) -> dict[str, str]:
|
||||
env = {"LEETCODE_COACH_LOG": str(session.log)}
|
||||
if session.cluster:
|
||||
env["LEETCODE_COACH_CLUSTER"] = session.cluster
|
||||
return env
|
||||
|
||||
|
||||
class CoachChat:
|
||||
"""Runs turns for whichever session is live."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ask: Callable[..., dict[str, Any]] = claude_cli.ask,
|
||||
local_chat: Callable[..., local_backend.LocalReply] = local_backend.chat,
|
||||
) -> None:
|
||||
"""Bind the claude call and the local model (both injectable)."""
|
||||
self._ask = ask
|
||||
self._local_chat = local_chat
|
||||
|
||||
def _bounded(self, messages: list[dict[str, str]]) -> local_backend.LocalReply:
|
||||
return self._local_chat(messages, timeout=TURN_TIMEOUT_S)
|
||||
|
||||
def open(self, session: Session) -> str:
|
||||
"""The coach's opening reply for a fresh session."""
|
||||
return self._claude(session, "/leetcode " + session.slug, resume=False)
|
||||
|
||||
def turn(self, session: Session, text: str) -> str:
|
||||
"""Kuhy's `text` in, the coach's raw reply out."""
|
||||
if session.local:
|
||||
return self._local(session, text, reason=None)
|
||||
return self._claude(session, text, resume=True)
|
||||
|
||||
def _claude(self, session: Session, text: str, *, resume: bool) -> str:
|
||||
args = ["--model", model()]
|
||||
if resume and session.claude_id:
|
||||
args += ["--resume", session.claude_id]
|
||||
reply = self._ask(args, text, timeout=TURN_TIMEOUT_S, env=_env(session))
|
||||
if claude_cli.is_limit_hit(reply):
|
||||
reason = str(reply.get("result") or "usage limit")
|
||||
_logger.warning("claude usage limit (%s): coaching on locally", reason)
|
||||
session.local = True
|
||||
return self._local(session, text, reason=reason)
|
||||
sid = reply.get("session_id")
|
||||
if reply.get("is_error") is True or not isinstance(sid, str) or not sid:
|
||||
_logger.warning("coach turn failed: %s", reply.get("_stderr") or reply)
|
||||
msg = "The coach did not answer; send it again."
|
||||
raise TurnError(msg, code="model_error", retry=True)
|
||||
session.claude_id = sid
|
||||
raw = str(reply.get("result", ""))
|
||||
session.history += [
|
||||
{"role": "user", "content": text},
|
||||
{"role": "assistant", "content": raw},
|
||||
]
|
||||
return raw
|
||||
|
||||
def _local(self, session: Session, text: str, *, reason: str | None) -> str:
|
||||
try:
|
||||
if session.local_coach is None:
|
||||
coach = local_coach.LocalCoach.start(
|
||||
session.statement or session.slug, session.history
|
||||
)
|
||||
coach.chat = self._bounded
|
||||
coach.leak_guard = session.gate.leaks
|
||||
session.local_coach = coach
|
||||
turn = session.local_coach.respond(text)
|
||||
except (
|
||||
urllib.error.URLError,
|
||||
OSError,
|
||||
local_backend.ContextOverflowError,
|
||||
) as exc:
|
||||
_logger.warning("local model could not answer either: %s", exc)
|
||||
why = reason or "claude usage limit"
|
||||
found = RESETS_RE.search(why)
|
||||
msg = "Claude is out of quota and the local model is down."
|
||||
raise TurnError(
|
||||
msg,
|
||||
code="claude_unavailable",
|
||||
reason=why,
|
||||
resets_at=found.group(1) if found else None,
|
||||
) from exc
|
||||
display, _cards = local_cards.split_cards(turn.text)
|
||||
return display
|
||||
208
leetcode_coach/leetcode_coach/relay/coach.py
Normal file
208
leetcode_coach/leetcode_coach/relay/coach.py
Normal file
@ -0,0 +1,208 @@
|
||||
"""The session backend every request runs against (DOCS-coach-relay.md 2-3).
|
||||
|
||||
One live session at a time. `handle` runs one validated request under the
|
||||
backend's lock (the lane already serialises; the lock also fences the
|
||||
Accepted ticker), then republishes the state: ``rev`` moves only when
|
||||
something visible changed, and listeners (the phone mirror) hear about it.
|
||||
Readers -- ``GET /coach/state`` and the heartbeat -- take the published copy
|
||||
and never wait for a model call in progress.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach import answer_key, claude_cli, local_backend, toolbox
|
||||
from leetcode_coach.relay import config, grading, protocol, runcode, steps, turns
|
||||
from leetcode_coach.relay.chat import CoachChat
|
||||
from leetcode_coach.relay.errors import CoachError
|
||||
from leetcode_coach.relay.probe import ACCEPTED, AcceptedProbe
|
||||
from leetcode_coach.relay.session import Session, grade_line, restore, save
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from leetcode_coach.phase import PhaseGate
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
ACCEPTED_NOTE: Final = "[app: step 4 passed, LeetCode shows an Accepted; step 5 open]"
|
||||
|
||||
|
||||
def _toolbox() -> toolbox.Toolbox:
|
||||
"""`toolbox.load_safe`, looked up per call (tests swap the fixture in)."""
|
||||
return toolbox.load_safe()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Deps:
|
||||
"""Everything the backend talks to; each one a test seam."""
|
||||
|
||||
state_dir: Path = field(default_factory=config.state_dir)
|
||||
ask: Callable[..., dict[str, Any]] = claude_cli.ask
|
||||
local_chat: Callable[..., local_backend.LocalReply] = local_backend.chat
|
||||
fetch: Callable[[str], tuple[str, str] | None] = turns.fetch_statement
|
||||
probe: AcceptedProbe = field(default_factory=AcceptedProbe)
|
||||
run_code: Callable[[str, str], dict[str, Any]] = runcode.run
|
||||
cards_dry_run: bool = field(default_factory=config.cards_dry_run)
|
||||
clock_ms: Callable[[], int] = config.now_ms
|
||||
answers: Callable[[], dict[str, answer_key.Answer]] = answer_key.load_answers
|
||||
box: Callable[[], toolbox.Toolbox] = field(default=_toolbox)
|
||||
|
||||
|
||||
class Coach:
|
||||
"""The live session, its published state, and the request handlers."""
|
||||
|
||||
def __init__(self, deps: Deps) -> None:
|
||||
"""Rebuild the live session from ``live.json`` if there is one."""
|
||||
self.deps = deps
|
||||
self.chat = CoachChat(deps.ask, deps.local_chat)
|
||||
self.lock = threading.RLock()
|
||||
self.listeners: list[Callable[[dict[str, Any], Session | None], None]] = []
|
||||
self.live_path = deps.state_dir / "live.json"
|
||||
restored = restore(self.live_path, self.gate_for)
|
||||
self.session: Session | None = None if restored is None else restored[0]
|
||||
self.rev = 0 if restored is None else restored[1] + 1
|
||||
self._seen = ""
|
||||
self._state: dict[str, Any] = {}
|
||||
self.publish()
|
||||
|
||||
def now(self) -> int:
|
||||
"""Unix ms."""
|
||||
return self.deps.clock_ms()
|
||||
|
||||
def gate_for(self, log: Path, slug: str) -> PhaseGate:
|
||||
"""A phase gate for one session's log (never ``PhaseGate.open``)."""
|
||||
return turns.gate_for(self, log, slug)
|
||||
|
||||
def state(self) -> dict[str, Any]:
|
||||
"""The published state (section 3)."""
|
||||
return self._state
|
||||
|
||||
def chat_after(self, after: str | None) -> dict[str, dict[str, Any]]:
|
||||
"""Chat lines after key `after` (all when None)."""
|
||||
session = self.session
|
||||
lines = {} if session is None else dict(session.chat)
|
||||
return {k: v for k, v in lines.items() if after is None or k > after}
|
||||
|
||||
def _build(self) -> dict[str, Any]:
|
||||
session = self.session
|
||||
if session is None:
|
||||
return {
|
||||
"schema": 1,
|
||||
"step": None,
|
||||
"rung": None,
|
||||
"revealed": False,
|
||||
"chat_count": 0,
|
||||
"session": None,
|
||||
"steps": [],
|
||||
}
|
||||
events = session.events()
|
||||
derived = steps.derive(events)
|
||||
current = steps.open_step(derived)
|
||||
step = None if session.ended is not None or current is None else current.n
|
||||
state: dict[str, Any] = {
|
||||
"schema": 1,
|
||||
"step": step,
|
||||
"rung": steps.rung(events),
|
||||
"revealed": steps.revealed(events),
|
||||
"chat_count": len(session.chat),
|
||||
"session": session.info(),
|
||||
"steps": [steps.view(s, session.probe, session.probed_at) for s in derived],
|
||||
}
|
||||
if step == grading.TOOLBOX_STEP:
|
||||
state["toolbox"] = [
|
||||
{"name": e.name, "properties": list(e.properties)}
|
||||
for e in self.deps.box().entries
|
||||
]
|
||||
return state
|
||||
|
||||
def publish(self) -> None:
|
||||
"""Rebuild the state; move ``rev`` and tell listeners if it changed."""
|
||||
with self.lock:
|
||||
body = self._build()
|
||||
seen = json.dumps(body, sort_keys=True)
|
||||
if seen == self._seen:
|
||||
return
|
||||
self._seen = seen
|
||||
self.rev += 1
|
||||
self._state = {**body, "rev": self.rev}
|
||||
if self.session is not None:
|
||||
save(self.live_path, self.session, self.rev)
|
||||
state, session = self._state, self.session
|
||||
for listener in self.listeners:
|
||||
listener(state, session)
|
||||
|
||||
def live(self, sid: object) -> Session:
|
||||
"""The live session `sid` names, or `not_live`."""
|
||||
session = self.session
|
||||
if session is None or session.ended is not None or session.sid != sid:
|
||||
msg = "That session is not the live one any more."
|
||||
raise CoachError(msg, code="not_live")
|
||||
return session
|
||||
|
||||
def end(self, session: Session, reason: str) -> None:
|
||||
"""Close `session` (it stays in the state, read-only)."""
|
||||
session.log_event("session_end", reason=reason, at=self.now())
|
||||
session.ended = self.now()
|
||||
session.end_reason = reason
|
||||
|
||||
def _dispatch(self, req: protocol.Request) -> dict[str, Any]:
|
||||
body = req.body
|
||||
if req.kind == "session_start":
|
||||
return turns.start(self, req)
|
||||
session = self.live(body.get("session"))
|
||||
if req.kind == "session_end":
|
||||
self.end(session, "ended")
|
||||
return {}
|
||||
if req.kind == "grade_step":
|
||||
return grading.grade_step(self, session, body)
|
||||
if req.kind == "run_code":
|
||||
return turns.run_code(self, session, str(body["code"]))
|
||||
text = {"hint": "hint", "reveal": "reveal"}.get(req.kind, body.get("text"))
|
||||
return turns.chat(self, session, str(text))
|
||||
|
||||
def handle(self, req: protocol.Request) -> dict[str, Any]:
|
||||
"""Run one request; its result (section 2)."""
|
||||
with self.lock:
|
||||
try:
|
||||
data = self._dispatch(req)
|
||||
except CoachError as exc:
|
||||
# An answer kuhy sees, not a crash; the journal says why too.
|
||||
_logger.warning("%s %s: %s (%s)", req.kind, req.rid, exc.code, exc)
|
||||
self.publish()
|
||||
return protocol.failure(
|
||||
exc.code, str(exc), at=self.now(), retry=exc.retry, **exc.extra
|
||||
)
|
||||
self.publish()
|
||||
return protocol.ok(data, at=self.now(), state_rev=self.rev)
|
||||
|
||||
def tick(self) -> None:
|
||||
"""Probe for step 4's Accepted, at most once a minute, off the lock."""
|
||||
session = self.session
|
||||
if session is None or session.ended is not None:
|
||||
return
|
||||
current = steps.open_step(steps.derive(session.events()))
|
||||
if current is None or current.n != steps.IMPLEMENT:
|
||||
return
|
||||
if not self.deps.probe.due():
|
||||
return
|
||||
status = self.deps.probe.check(session.slug, session.started)
|
||||
with self.lock:
|
||||
if self.session is not session or session.ended is not None:
|
||||
return
|
||||
session.probe, session.probed_at = status, self.now()
|
||||
if status == ACCEPTED:
|
||||
at = self.now()
|
||||
event = {"step": 4, "grade": "pass", "attempt": 1, "forced": False}
|
||||
event |= {"via": "accepted", "feedback": "", "at": at}
|
||||
session.log_event("grade", **event)
|
||||
kind, text = grade_line(event)
|
||||
session.add("system", kind, text, at)
|
||||
session.notes.append(ACCEPTED_NOTE)
|
||||
self.publish()
|
||||
48
leetcode_coach/leetcode_coach/relay/config.py
Normal file
48
leetcode_coach/leetcode_coach/relay/config.py
Normal file
@ -0,0 +1,48 @@
|
||||
"""Where the relay keeps its files, and its test seams (contract section 7).
|
||||
|
||||
Every default is resolved at call time, so a test that sets an env var after
|
||||
import still moves the path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
DIR_ENV: Final = "LEETCODE_COACH_RELAY_DIR"
|
||||
PORT_ENV: Final = "LEETCODE_COACH_RELAY_PORT"
|
||||
RTDB_ENV: Final = "LEETCODE_COACH_RTDB_URL"
|
||||
CARDS_DRY_RUN_ENV: Final = "LEETCODE_COACH_CARDS_DRY_RUN"
|
||||
DEFAULT_DIR: Final = Path.home() / ".local" / "share" / "leetcode-coach" / "relay"
|
||||
HOST: Final = "127.0.0.1"
|
||||
DEFAULT_PORT: Final = 8777
|
||||
APP_NAME: Final = "leetcode_coach_relay"
|
||||
"""The crdt_sync app the relay signs in as (its own token cache)."""
|
||||
|
||||
|
||||
def state_dir() -> Path:
|
||||
"""``~/.local/share/leetcode-coach/relay`` unless ``$LEETCODE_COACH_RELAY_DIR``."""
|
||||
override = os.environ.get(DIR_ENV)
|
||||
return Path(override) if override else DEFAULT_DIR
|
||||
|
||||
|
||||
def port() -> int:
|
||||
"""The loopback port (8777 unless ``$LEETCODE_COACH_RELAY_PORT``)."""
|
||||
return int(os.environ.get(PORT_ENV) or DEFAULT_PORT)
|
||||
|
||||
|
||||
def rtdb_url() -> str | None:
|
||||
"""A database URL that skips Firebase sign-in (the fake RTDB), if set."""
|
||||
return os.environ.get(RTDB_ENV) or None
|
||||
|
||||
|
||||
def cards_dry_run() -> bool:
|
||||
"""Whether step 5 only validates its cards (the live smoke run)."""
|
||||
return os.environ.get(CARDS_DRY_RUN_ENV) == "1"
|
||||
|
||||
|
||||
def now_ms() -> int:
|
||||
"""Wall-clock unix milliseconds."""
|
||||
return time.time_ns() // 1_000_000
|
||||
107
leetcode_coach/leetcode_coach/relay/done.py
Normal file
107
leetcode_coach/leetcode_coach/relay/done.py
Normal file
@ -0,0 +1,107 @@
|
||||
"""The done-map: every finished rid and its result (DOCS-coach-relay.md 1).
|
||||
|
||||
A client that lost the answer sends the SAME rid again, and the relay must
|
||||
re-answer it, never re-run it: a re-run chat turn would spend a model call
|
||||
and post a second reply. So every result -- ``bad_request`` and
|
||||
``rate_limited`` included -- lands here before anyone is told. Kept in
|
||||
``requests_done.jsonl`` for 30 days, so a relay restart still knows; a
|
||||
desktop result is forgotten after 24 h (the wrapper polls for seconds).
|
||||
|
||||
Appends are one ``os.write`` on an ``O_APPEND`` descriptor, like the session
|
||||
log. The file is compacted at start-up, when expired lines are dropped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
DAY_MS: Final = 24 * 3600 * 1000
|
||||
KEEP_MS: Final = 30 * DAY_MS
|
||||
DESKTOP_KEEP_MS: Final = DAY_MS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Done:
|
||||
"""One finished request."""
|
||||
|
||||
rid: str
|
||||
at: int
|
||||
transport: str
|
||||
result: dict[str, Any]
|
||||
|
||||
|
||||
def _parse(line: str) -> Done | None:
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
return Done(
|
||||
str(raw["rid"]), int(raw["at"]), str(raw["transport"]), dict(raw["result"])
|
||||
)
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
_logger.warning("done-map line skipped: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
class DoneMap:
|
||||
"""Finished rids, in memory and on disk."""
|
||||
|
||||
def __init__(self, path: Path, clock_ms: Callable[[], int]) -> None:
|
||||
"""Load `path`, dropping (and compacting away) expired results."""
|
||||
self._path = path
|
||||
self._clock = clock_ms
|
||||
self._lock = threading.Lock()
|
||||
self._done: dict[str, Done] = {}
|
||||
lines = path.read_text(encoding="utf-8").splitlines() if path.is_file() else []
|
||||
cutoff = clock_ms() - KEEP_MS
|
||||
for line in lines:
|
||||
done = _parse(line) if line.strip() else None
|
||||
if done is not None and done.at >= cutoff:
|
||||
self._done[done.rid] = done
|
||||
if len(self._done) != len(lines):
|
||||
self._rewrite()
|
||||
|
||||
def _line(self, done: Done) -> bytes:
|
||||
record = {
|
||||
"rid": done.rid,
|
||||
"at": done.at,
|
||||
"transport": done.transport,
|
||||
"result": done.result,
|
||||
}
|
||||
return (json.dumps(record, ensure_ascii=False) + "\n").encode()
|
||||
|
||||
def _rewrite(self) -> None:
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = self._path.with_suffix(".tmp")
|
||||
tmp.write_bytes(b"".join(self._line(d) for d in self._done.values()))
|
||||
tmp.replace(self._path)
|
||||
|
||||
def put(self, rid: str, transport: str, result: dict[str, Any]) -> None:
|
||||
"""Record `rid`'s result (before any client is told about it)."""
|
||||
done = Done(rid, self._clock(), transport, result)
|
||||
with self._lock:
|
||||
self._done[rid] = done
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = os.open(self._path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
|
||||
try:
|
||||
os.write(fd, self._line(done))
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def get(self, rid: str) -> Done | None:
|
||||
"""The finished request, unless it expired (24 h for the desktop)."""
|
||||
with self._lock:
|
||||
done = self._done.get(rid)
|
||||
if done is None:
|
||||
return None
|
||||
keep = DESKTOP_KEEP_MS if done.transport == "desktop" else KEEP_MS
|
||||
return done if self._clock() - done.at <= keep else None
|
||||
16
leetcode_coach/leetcode_coach/relay/errors.py
Normal file
16
leetcode_coach/leetcode_coach/relay/errors.py
Normal file
@ -0,0 +1,16 @@
|
||||
"""The one exception a request handler raises to answer with an error result."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class CoachError(Exception):
|
||||
"""Answer the request with ``code``; the message is shown to kuhy."""
|
||||
|
||||
def __init__(
|
||||
self, message: str, *, code: str, retry: bool = False, **extra: object
|
||||
) -> None:
|
||||
"""Keep the result fields (section 2's error codes)."""
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.retry = retry
|
||||
self.extra = extra
|
||||
127
leetcode_coach/leetcode_coach/relay/examples.py
Normal file
127
leetcode_coach/leetcode_coach/relay/examples.py
Normal file
@ -0,0 +1,127 @@
|
||||
"""The statement's worked examples and the starter's function name (run_code).
|
||||
|
||||
The statement is ``fetch_problem.render(q, "javascript")`` markdown. LeetCode
|
||||
writes an example in three shapes, all handled by one line scan: bold labels
|
||||
on plain lines (``**Input:** nums = [3,0,1]``), the same with the values in
|
||||
code spans, and the same inside a ``` fence. An ``Input:`` line is
|
||||
``name = <json>`` pairs split on top-level commas; the following ``Output:``
|
||||
line closes the example. A value that is not JSON (a linked list drawn as
|
||||
``1->2``, a design problem's call list) skips that example with a warning:
|
||||
running kuhy's code against a guessed input would report a wrong "got".
|
||||
|
||||
Examples are numbered by their place in the statement, so "example 3" is the
|
||||
one kuhy reads as Example 3 even when an earlier one was skipped. Only the
|
||||
body counts: the starter code and the site hints come after the first
|
||||
``## `` heading and are never scanned for examples.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Final
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
_NAME_RES: Final = (
|
||||
re.compile(r"\b(?:var|let|const)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:function\b|\()"),
|
||||
re.compile(r"\bfunction\s+([A-Za-z_$][\w$]*)\s*\("),
|
||||
)
|
||||
_LABEL: Final = re.compile(r"^(Input|Output)\s*:\s*(.*)$")
|
||||
_PAIR: Final = re.compile(r"^([A-Za-z_]\w*)\s*=\s*(.+)$", re.DOTALL)
|
||||
_STARTER: Final = "## Starter code"
|
||||
_OPENERS: Final = {"[": "]", "{": "}", "(": ")"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Example:
|
||||
"""One worked example: its number, the call's arguments, the answer."""
|
||||
|
||||
n: int
|
||||
args: tuple[object, ...]
|
||||
expected: object
|
||||
|
||||
|
||||
def function_name(statement: str) -> str | None:
|
||||
"""The function the starter code declares (starter section first)."""
|
||||
starter = statement.split(_STARTER, 1)
|
||||
for text in (starter[-1], statement):
|
||||
for pattern in _NAME_RES:
|
||||
if match := pattern.search(text):
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def split_top_level(text: str) -> list[str]:
|
||||
"""``text`` split on commas outside brackets and string literals."""
|
||||
parts: list[str] = []
|
||||
depth = 0
|
||||
quoted = escaped = False
|
||||
start = 0
|
||||
for i, ch in enumerate(text):
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quoted:
|
||||
escaped = ch == "\\"
|
||||
quoted = ch != '"'
|
||||
elif ch == '"':
|
||||
quoted = True
|
||||
elif ch in _OPENERS:
|
||||
depth += 1
|
||||
elif ch in _OPENERS.values():
|
||||
depth -= 1
|
||||
elif ch == "," and depth == 0:
|
||||
parts.append(text[start:i])
|
||||
start = i + 1
|
||||
parts.append(text[start:])
|
||||
return [p.strip() for p in parts]
|
||||
|
||||
|
||||
def _arguments(text: str) -> tuple[object, ...]:
|
||||
"""The values of ``a = 1, b = [2]``; ValueError when one is not JSON."""
|
||||
values: list[object] = []
|
||||
for part in split_top_level(text):
|
||||
match = _PAIR.match(part)
|
||||
if match is None:
|
||||
msg = f"not a name = value pair: {part[:40]!r}"
|
||||
raise ValueError(msg)
|
||||
values.append(json.loads(match.group(2)))
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def _clean(line: str) -> str:
|
||||
"""A line without markdown emphasis, code ticks and (nb)spaces around it."""
|
||||
return line.replace("**", "").replace("`", "").strip()
|
||||
|
||||
|
||||
def _labels(body: str) -> list[tuple[str, str]]:
|
||||
"""Every ``(Input|Output, value)`` line of the body, in order."""
|
||||
matches = (_LABEL.match(_clean(raw)) for raw in body.splitlines())
|
||||
return [(m.group(1), m.group(2).strip()) for m in matches if m is not None]
|
||||
|
||||
|
||||
def _body(statement: str) -> str:
|
||||
"""The statement above its first ``## `` section (starter, hints)."""
|
||||
return re.split(r"^## ", statement, maxsplit=1, flags=re.MULTILINE)[0]
|
||||
|
||||
|
||||
def parse(statement: str) -> tuple[str | None, list[Example]]:
|
||||
"""The starter's function name and every example that parses."""
|
||||
found: list[Example] = []
|
||||
pending: str | None = None
|
||||
number = 0
|
||||
for label, value in _labels(_body(statement)):
|
||||
if label == "Input":
|
||||
pending = value
|
||||
continue
|
||||
if pending is None:
|
||||
continue
|
||||
number += 1
|
||||
try:
|
||||
found.append(Example(number, _arguments(pending), json.loads(value)))
|
||||
except ValueError as exc:
|
||||
_logger.warning("example %d skipped, not runnable: %s", number, exc)
|
||||
pending = None
|
||||
return function_name(statement), found
|
||||
141
leetcode_coach/leetcode_coach/relay/graders.py
Normal file
141
leetcode_coach/leetcode_coach/relay/graders.py
Normal file
@ -0,0 +1,141 @@
|
||||
"""Steps 1 and 5 on a cheap model; the feedback rules every grade obeys.
|
||||
|
||||
Same route as the requirement checker (`req_check.ask_models`): haiku with no
|
||||
tools, then Ollama, and when neither answers the step passes ``unverified``
|
||||
(fail open: a grader outage must never wall kuhy in). The answer key goes to
|
||||
the grader and nowhere else, so feedback is checked mechanically before it is
|
||||
shown: it may not name a toolbox entry kuhy has not named yet (until the
|
||||
first code, `leak.leaked_names`), and it may not quote the key (any run of
|
||||
`QUOTE_WORDS` words from it). A failing feedback is replaced by the plain
|
||||
missing-property line.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach import leak, req_check
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from leetcode_coach.answer_key import Answer
|
||||
from leetcode_coach.toolbox import Toolbox
|
||||
|
||||
GRADES: Final = ("pass", "partial", "fail")
|
||||
QUOTE_WORDS: Final = 5
|
||||
POSTMORTEM_STEP: Final = 5
|
||||
UNVERIFIED: Final = "Not checked: the grader could not be reached, so this step counts."
|
||||
CONSTRAINT_HINT: Final = "Name the limit that binds and say what it rules out."
|
||||
_RULES: Final = (
|
||||
'Reply with ONE JSON object and nothing else: {"grade": "pass" | '
|
||||
'"partial" | "fail", "feedback": "<one or two sentences>"}. The feedback '
|
||||
"names what is missing as a property of a good answer. Never name an "
|
||||
"operation, a data structure or an algorithm, never give the solution, and "
|
||||
"never quote or paraphrase the reference: it is secret."
|
||||
)
|
||||
CONSTRAINT_SYSTEM: Final = (
|
||||
"You grade one step of a student's method for a programming problem: "
|
||||
"naming the binding constraint and what it forces. pass = the answer names "
|
||||
"the limit that binds and what it rules out or forces; partial = only one "
|
||||
"of the two; fail = neither. " + _RULES
|
||||
)
|
||||
POSTMORTEM_SYSTEM: Final = (
|
||||
"You grade a student's post-mortem, written after solving a programming "
|
||||
"problem: which properties of the operation did the work, and how they "
|
||||
"could have found it from the constraints. pass = it names the key "
|
||||
"properties and at least one of the reference points; partial = some of "
|
||||
"them; fail = none. Property names from the key list may be used in the "
|
||||
"feedback. " + _RULES
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Grade:
|
||||
"""One grader answer."""
|
||||
|
||||
grade: str
|
||||
feedback: str
|
||||
via: str
|
||||
|
||||
|
||||
def _valid(obj: dict[str, Any] | None) -> Grade | None:
|
||||
if obj is None or obj.get("grade") not in GRADES:
|
||||
return None
|
||||
feedback = obj.get("feedback", "")
|
||||
if not isinstance(feedback, str):
|
||||
return None
|
||||
return Grade(str(obj["grade"]), feedback.strip(), "")
|
||||
|
||||
|
||||
def _prompt(step: int, statement: str, answer: str, key: Answer) -> str:
|
||||
if step == 1:
|
||||
reference = f"## Reference rule (secret)\n{key.design_rule}"
|
||||
else:
|
||||
points = "\n".join(f"- {p}" for p in key.postmortem_points)
|
||||
reference = (
|
||||
f"## Reference (secret)\nRule: {key.design_rule}\n"
|
||||
f"Key properties: {', '.join(key.key_requirements)}\nPoints:\n{points}"
|
||||
)
|
||||
return (
|
||||
f"## Problem\n{statement.strip()}\n\n{reference}\n\n"
|
||||
f"## Student's answer\n{answer.strip()}\n"
|
||||
)
|
||||
|
||||
|
||||
def model_grade(
|
||||
step: int,
|
||||
statement: str,
|
||||
answer: str,
|
||||
key: Answer,
|
||||
ask: Callable[..., dict[str, Any]] | None,
|
||||
) -> Grade:
|
||||
"""Grade step 1 (constraint) or 5 (post-mortem); ``unverified`` pass if down."""
|
||||
system = CONSTRAINT_SYSTEM if step == 1 else POSTMORTEM_SYSTEM
|
||||
got = req_check.ask_models(
|
||||
system, _prompt(step, statement, answer, key), _valid, ask
|
||||
)
|
||||
if got is None:
|
||||
return Grade("pass", UNVERIFIED, "unverified")
|
||||
grade, via = got
|
||||
return Grade(grade.grade, grade.feedback, via)
|
||||
|
||||
|
||||
def _words(text: str) -> list[str]:
|
||||
return re.findall(r"[a-z0-9]+", text.lower())
|
||||
|
||||
|
||||
def quotes_key(feedback: str, key: Answer) -> bool:
|
||||
"""Whether `feedback` repeats `QUOTE_WORDS` consecutive words of the key."""
|
||||
said = " ".join(_words(feedback))
|
||||
for source in (key.design_rule, *key.postmortem_points):
|
||||
words = _words(source)
|
||||
for i in range(len(words) - QUOTE_WORDS + 1):
|
||||
if " ".join(words[i : i + QUOTE_WORDS]) in said:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def fallback(step: int, key: Answer) -> str:
|
||||
"""The plain missing-property line a rejected feedback becomes."""
|
||||
if step == POSTMORTEM_STEP:
|
||||
return "Say which properties did the work: " + ", ".join(key.key_requirements)
|
||||
return CONSTRAINT_HINT
|
||||
|
||||
|
||||
def safe_feedback( # pylint: disable=too-many-arguments
|
||||
feedback: str,
|
||||
*,
|
||||
step: int,
|
||||
key: Answer,
|
||||
box: Toolbox,
|
||||
kuhy_texts: list[str],
|
||||
code_seen: bool,
|
||||
) -> str:
|
||||
"""`feedback`, or `fallback` when it names an entry early or quotes the key."""
|
||||
early = not code_seen and leak.leaked_names(feedback, kuhy_texts, box)
|
||||
if early or quotes_key(feedback, key):
|
||||
return fallback(step, key)
|
||||
return feedback
|
||||
154
leetcode_coach/leetcode_coach/relay/grading.py
Normal file
154
leetcode_coach/leetcode_coach/relay/grading.py
Normal file
@ -0,0 +1,154 @@
|
||||
"""``grade_step``: grade steps 1, 2, 3 and 5, or let kuhy continue (force).
|
||||
|
||||
1 and 5 go to the model grader (`graders`), 2 to the CLI's own phase gate
|
||||
(`PhaseGate.screen` logs ``req`` + ``req_check``), 3 to the mechanical
|
||||
toolbox grader. Every graded try is one ``grade`` event; the step machine
|
||||
reads them back (`steps`). The grade reaches the coach as a note on its next
|
||||
turn and the chat as a ``system`` line at once. A step-5 pass builds the
|
||||
cards and ends the session (``done``); so does forcing step 5, without cards.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach import leak, phase
|
||||
from leetcode_coach.relay import cards, graders, steps, toolbox_grade, turns
|
||||
from leetcode_coach.relay.errors import CoachError
|
||||
from leetcode_coach.relay.session import grade_line
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from leetcode_coach.answer_key import Answer
|
||||
from leetcode_coach.relay.coach import Coach
|
||||
from leetcode_coach.relay.session import Session
|
||||
from leetcode_coach.toolbox import Toolbox
|
||||
|
||||
REQ_STEP: Final = 2
|
||||
TOOLBOX_STEP: Final = 3
|
||||
POSTMORTEM_STEP: Final = 5
|
||||
READS_AS_CODE: Final = (
|
||||
"This reads as code. List the properties the operation must have, in words."
|
||||
)
|
||||
NAMES_A_TOOL: Final = (
|
||||
"This names an operation or a structure. List the properties it must have "
|
||||
"instead, without naming it."
|
||||
)
|
||||
|
||||
|
||||
def _requirements(session: Session, answer: str, box: Toolbox) -> tuple[str, str, str]:
|
||||
"""Step 2 through the phase gate; a list that is code or names a tool fails."""
|
||||
text = answer.strip()
|
||||
words = phase.REQ_RE.sub("", text, count=1).strip()
|
||||
if phase.regex_verdict(words) is True:
|
||||
return "fail", READS_AS_CODE, "toolbox"
|
||||
if leak.leaked_names(words, [], box):
|
||||
return "fail", NAMES_A_TOOL, "toolbox"
|
||||
session.gate.screen(text if phase.REQ_RE.match(text) else f"req: {text}")
|
||||
check = [e for e in session.events() if e.get("event") == "req_check"][-1]
|
||||
via = str(check.get("via") or "unverified")
|
||||
if check.get("verdict") == "pass":
|
||||
return "pass", "", via
|
||||
if check.get("verdict") == "unverified":
|
||||
return "pass", graders.UNVERIFIED, "unverified"
|
||||
missing = [str(m) for m in check.get("missing", []) if m in box.properties]
|
||||
lines = [f"- {phase.property_line(m, box)}" for m in missing]
|
||||
return "partial", "\n".join(["It does not yet say:", *lines]), via
|
||||
|
||||
|
||||
def _toolbox(picks: list[str], key: Answer, box: Toolbox) -> tuple[str, str, str]:
|
||||
unknown = [p for p in picks if toolbox_grade.resolve(p, box) is None]
|
||||
if unknown:
|
||||
msg = "Not in the toolbox: " + ", ".join(unknown)
|
||||
raise CoachError(msg, code="bad_request")
|
||||
grade, feedback = toolbox_grade.grade(picks, key, box)
|
||||
return grade, feedback, "toolbox"
|
||||
|
||||
|
||||
def _graded(
|
||||
coach: Coach, session: Session, n: int, body: dict[str, Any]
|
||||
) -> tuple[str, str, str]:
|
||||
key = coach.deps.answers()[session.slug]
|
||||
box = coach.deps.box()
|
||||
if n == TOOLBOX_STEP:
|
||||
return _toolbox(list(body["picks"]), key, box)
|
||||
if n == REQ_STEP:
|
||||
return _requirements(session, str(body["answer"]), box)
|
||||
text = turns.statement(coach, session) or session.slug
|
||||
got = graders.model_grade(n, text, str(body["answer"]), key, coach.deps.ask)
|
||||
users = [str(v["text"]) for v in session.chat.values() if v["role"] == "kuhy"]
|
||||
feedback = graders.safe_feedback(
|
||||
got.feedback,
|
||||
step=n,
|
||||
key=key,
|
||||
box=box,
|
||||
kuhy_texts=users,
|
||||
code_seen=session.gate.code_seen(),
|
||||
)
|
||||
return got.grade, feedback, got.via
|
||||
|
||||
|
||||
def _record(session: Session, event: dict[str, Any]) -> int | None:
|
||||
"""Log one grade event, show it, note it for the coach; the step now open."""
|
||||
session.log_event("grade", **event)
|
||||
kind, text = grade_line(event)
|
||||
session.add("system", kind, text, int(event["at"]))
|
||||
current = steps.open_step(steps.derive(session.events()))
|
||||
nxt = None if current is None else current.n
|
||||
what = "continued without a pass" if event["forced"] else f"graded {event['grade']}"
|
||||
opened = "all steps done" if nxt is None else f"step {nxt} open"
|
||||
note = f"[app: step {event['step']} {what} on try {event['attempt']}; {opened}]"
|
||||
session.notes.append(note)
|
||||
return nxt
|
||||
|
||||
|
||||
def _force(coach: Coach, session: Session, step: steps.Step) -> dict[str, Any]:
|
||||
if not step.can_force:
|
||||
msg = f"You can continue after {steps.FORCE_AFTER} tries without a pass."
|
||||
raise CoachError(msg, code="cannot_force")
|
||||
last = step.last or {}
|
||||
event = {"step": step.n, "grade": last.get("grade") or "fail"}
|
||||
event |= {"attempt": step.attempts, "forced": True, "feedback": ""}
|
||||
event |= {"via": last.get("via") or "unverified", "at": coach.now()}
|
||||
nxt = _record(session, event)
|
||||
if step.n == POSTMORTEM_STEP:
|
||||
coach.end(session, "done")
|
||||
return {"step": step.n, "forced": True, "next_step": nxt}
|
||||
|
||||
|
||||
def grade_step(coach: Coach, session: Session, body: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Grade (or force) ``body.step`` of the live session."""
|
||||
n = int(body["step"])
|
||||
step = steps.derive(session.events())[n - 1]
|
||||
if body.get("force") is True and n == steps.IMPLEMENT:
|
||||
msg = "Step 4 passes only when LeetCode shows an Accepted."
|
||||
raise CoachError(msg, code="cannot_force")
|
||||
if step.status == "locked":
|
||||
msg = f"Step {n} opens once step {n - 1} is passed or continued."
|
||||
raise CoachError(msg, code="step_locked")
|
||||
if step.done:
|
||||
msg = f"Step {n} is already done."
|
||||
raise CoachError(msg, code="step_done")
|
||||
if body.get("force") is True:
|
||||
return _force(coach, session, step)
|
||||
attempt = step.attempts + 1
|
||||
grade, feedback, via = _graded(coach, session, n, body)
|
||||
event = {"step": n, "grade": grade, "attempt": attempt, "forced": False}
|
||||
event |= {"via": via, "feedback": feedback, "at": coach.now()}
|
||||
nxt = _record(session, event)
|
||||
after = steps.derive(session.events())[n - 1]
|
||||
data: dict[str, Any] = {
|
||||
"step": n,
|
||||
"grade": grade,
|
||||
"feedback": feedback,
|
||||
"attempt": attempt,
|
||||
"can_force": after.can_force,
|
||||
"via": via,
|
||||
"next_step": nxt,
|
||||
}
|
||||
if n == POSTMORTEM_STEP and grade == "pass":
|
||||
key = coach.deps.answers()[session.slug]
|
||||
data["cards"] = cards.build(
|
||||
session, key.entries, dry_run=coach.deps.cards_dry_run
|
||||
)
|
||||
coach.end(session, "done")
|
||||
return data
|
||||
162
leetcode_coach/leetcode_coach/relay/http_api.py
Normal file
162
leetcode_coach/leetcode_coach/relay/http_api.py
Normal file
@ -0,0 +1,162 @@
|
||||
"""The desktop's loopback API on 127.0.0.1:8777 (DOCS-coach-relay.md 1, 4).
|
||||
|
||||
Reached only through the app's wrapper on :8776, never by the browser. It
|
||||
binds the loopback address, refuses a ``Host`` other than its own (DNS
|
||||
rebinding) and takes POST bodies only as ``application/json``, so a
|
||||
cross-origin POST needs a preflight that is never granted. No route takes a
|
||||
path, a command or a URL; a request body goes to `protocol.parse` and from
|
||||
there to the lane, like the phone's.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from leetcode_coach.relay import protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from leetcode_coach.relay.coach import Coach
|
||||
from leetcode_coach.relay.lane import Lane
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
PREFIX: Final = "/coach/"
|
||||
REQUESTS: Final = "/coach/requests"
|
||||
CHAT_KEY_RE_LEN: Final = 5
|
||||
|
||||
|
||||
class App(Protocol):
|
||||
"""What the API serves: the lane, the backend and the heartbeat."""
|
||||
|
||||
lane: Lane
|
||||
coach: Coach
|
||||
|
||||
def heartbeat(self) -> dict[str, Any]:
|
||||
"""The heartbeat object, ``beat`` as unix ms."""
|
||||
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
"""A threading server that knows its relay."""
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, address: tuple[str, int], app: App) -> None:
|
||||
"""Bind `address` and keep `app` for the handlers."""
|
||||
super().__init__(address, Handler)
|
||||
self.app = app
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
"""The four GET routes and the one POST."""
|
||||
|
||||
server: Server
|
||||
server_version = "leetcode-coach-relay"
|
||||
|
||||
def log_message(self, fmt: str, /, *args: object) -> None:
|
||||
"""Request lines to the journal at debug, not stderr."""
|
||||
_logger.debug("%s - %s", self.address_string(), fmt % args)
|
||||
|
||||
def _send(self, status: HTTPStatus, payload: object) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _error(self, status: HTTPStatus, code: str, error: str) -> None:
|
||||
self._send(status, {"ok": False, "code": code, "error": error})
|
||||
|
||||
def _host_ok(self) -> bool:
|
||||
port = self.server.server_address[1]
|
||||
host = self.headers.get("Host", "")
|
||||
if host in {f"127.0.0.1:{port}", f"localhost:{port}"}:
|
||||
return True
|
||||
_logger.warning("refused a request for Host %r", host)
|
||||
self._error(HTTPStatus.FORBIDDEN, "bad_host", "loopback only")
|
||||
return False
|
||||
|
||||
def do_GET(self) -> None:
|
||||
"""State, heartbeat, chat, or one request's result."""
|
||||
if not self._host_ok():
|
||||
return
|
||||
url = urlsplit(self.path)
|
||||
app = self.server.app
|
||||
if url.path == "/coach/state":
|
||||
self._send(HTTPStatus.OK, app.coach.state())
|
||||
elif url.path == "/coach/heartbeat":
|
||||
self._send(HTTPStatus.OK, app.heartbeat())
|
||||
elif url.path == "/coach/chat":
|
||||
after = parse_qs(url.query).get("after", [None])[0]
|
||||
self._send(HTTPStatus.OK, app.coach.chat_after(after))
|
||||
elif url.path.startswith(REQUESTS + "/"):
|
||||
rid = url.path.removeprefix(REQUESTS + "/")
|
||||
if not protocol.RID_RE.fullmatch(rid) or not app.lane.known(rid):
|
||||
self._error(HTTPStatus.NOT_FOUND, "unknown_rid", "no such request")
|
||||
return
|
||||
self._send(HTTPStatus.OK, {"result": app.lane.result(rid)})
|
||||
else:
|
||||
self._error(HTTPStatus.NOT_FOUND, "not_found", "no such route")
|
||||
|
||||
def _body(self) -> object:
|
||||
"""The decoded JSON body, or None after an error answer was sent."""
|
||||
ctype = self.headers.get("Content-Type", "").split(";")[0].strip()
|
||||
if ctype != "application/json":
|
||||
self._error(
|
||||
HTTPStatus.UNSUPPORTED_MEDIA_TYPE, "bad_request", "JSON bodies only"
|
||||
)
|
||||
return None
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
length = -1
|
||||
_logger.warning("a POST with a bad Content-Length")
|
||||
if length > protocol.MAX_REQ_BYTES:
|
||||
self._error(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, "bad_request", "too big")
|
||||
return None
|
||||
if length < 0:
|
||||
self._error(HTTPStatus.BAD_REQUEST, "bad_request", "bad length")
|
||||
return None
|
||||
try:
|
||||
return json.loads(self.rfile.read(length) or b"null")
|
||||
except ValueError as exc:
|
||||
_logger.warning("a POST body that is not JSON: %s", exc)
|
||||
self._error(HTTPStatus.BAD_REQUEST, "bad_request", "not JSON")
|
||||
return None
|
||||
|
||||
def do_POST(self) -> None:
|
||||
"""Queue one request: 202, or 200 with its result when already known."""
|
||||
if not self._host_ok():
|
||||
return
|
||||
if urlsplit(self.path).path != REQUESTS:
|
||||
self._error(HTTPStatus.NOT_FOUND, "not_found", "no such route")
|
||||
return
|
||||
raw = self._body()
|
||||
if raw is None:
|
||||
return
|
||||
lane = self.server.app.lane
|
||||
try:
|
||||
req = protocol.parse(raw, "desktop")
|
||||
except protocol.BadRidError as exc:
|
||||
_logger.warning("desktop request refused: %s", exc)
|
||||
self._error(HTTPStatus.BAD_REQUEST, "bad_request", str(exc))
|
||||
return
|
||||
except protocol.BadRequestError as exc:
|
||||
_logger.warning("bad desktop request: %s", exc)
|
||||
rid = str(raw["rid"]) if isinstance(raw, dict) else ""
|
||||
if (done := lane.result(rid)) is None:
|
||||
done = protocol.failure("bad_request", str(exc), at=lane.now())
|
||||
lane.finish(rid, "desktop", done)
|
||||
self._send(HTTPStatus.OK, done)
|
||||
return
|
||||
result = lane.submit(req)
|
||||
if result is None:
|
||||
self._send(HTTPStatus.ACCEPTED, {"rid": req.rid})
|
||||
else:
|
||||
self._send(HTTPStatus.OK, result)
|
||||
149
leetcode_coach/leetcode_coach/relay/lane.py
Normal file
149
leetcode_coach/leetcode_coach/relay/lane.py
Normal file
@ -0,0 +1,149 @@
|
||||
"""The one paced serial lane both transports feed (DOCS-coach-relay.md 1).
|
||||
|
||||
FIFO by arrival; one request runs at a time and none overtakes an earlier
|
||||
one. At most one request *starts* every `pace_s` (3 s), globally across the
|
||||
desktop and the phone; the rest wait. A request arriving while `max_queued`
|
||||
(5) already wait is answered ``rate_limited`` and never runs.
|
||||
|
||||
Every result goes into the done-map before any listener hears of it, so a
|
||||
client that resends a rid is re-answered, never re-run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach.relay import protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from leetcode_coach.relay.done import DoneMap
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
PACE_S: Final = 3.0
|
||||
MAX_QUEUED: Final = 5
|
||||
|
||||
|
||||
class Lane: # pylint: disable=too-many-instance-attributes
|
||||
"""Queue, pacing and the worker loop around one request handler."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
handle: Callable[[protocol.Request], dict[str, Any]],
|
||||
done: DoneMap,
|
||||
*,
|
||||
clock_ms: Callable[[], int],
|
||||
monotonic: Callable[[], float] = time.monotonic,
|
||||
pace_s: float = PACE_S,
|
||||
max_queued: int = MAX_QUEUED,
|
||||
) -> None:
|
||||
"""Bind the handler; nothing runs until `run`."""
|
||||
self._handle = handle
|
||||
self.done = done
|
||||
self._clock = clock_ms
|
||||
self._monotonic = monotonic
|
||||
self._pace = pace_s
|
||||
self._max = max_queued
|
||||
self._cv = threading.Condition()
|
||||
self._queue: deque[protocol.Request] = deque()
|
||||
self._last_start = float("-inf")
|
||||
self.busy: str | None = None
|
||||
self.on_result: list[Callable[[protocol.Request, dict[str, Any]], None]] = []
|
||||
self.on_busy: list[Callable[[str | None], None]] = []
|
||||
|
||||
def now(self) -> int:
|
||||
"""Unix ms, by the relay's clock."""
|
||||
return self._clock()
|
||||
|
||||
def known(self, rid: str) -> bool:
|
||||
"""Whether `rid` is queued, running or finished."""
|
||||
with self._cv:
|
||||
waiting = rid == self.busy or any(r.rid == rid for r in self._queue)
|
||||
return waiting or self.done.get(rid) is not None
|
||||
|
||||
def result(self, rid: str) -> dict[str, Any] | None:
|
||||
"""The finished result of `rid`, or None while it waits or runs."""
|
||||
done = self.done.get(rid)
|
||||
return None if done is None else done.result
|
||||
|
||||
def finish(self, rid: str, transport: str, result: dict[str, Any]) -> None:
|
||||
"""Store a result that never ran (``bad_request``, ``rate_limited``)."""
|
||||
self.done.put(rid, transport, result)
|
||||
|
||||
def submit(self, req: protocol.Request) -> dict[str, Any] | None:
|
||||
"""Queue `req`; its result now if it is already known, else None."""
|
||||
if (done := self.done.get(req.rid)) is not None:
|
||||
return done.result
|
||||
with self._cv:
|
||||
if req.rid == self.busy or any(r.rid == req.rid for r in self._queue):
|
||||
return None
|
||||
if len(self._queue) >= self._max:
|
||||
wait_ms = int(self._pace * 1000 * len(self._queue))
|
||||
result = protocol.failure(
|
||||
"rate_limited",
|
||||
"The PC is busy with earlier requests; send again shortly.",
|
||||
at=self._clock(),
|
||||
retry=True,
|
||||
retry_after_ms=wait_ms,
|
||||
)
|
||||
self.finish(req.rid, req.transport, result)
|
||||
return result
|
||||
self._queue.append(req)
|
||||
self._cv.notify_all()
|
||||
return None
|
||||
|
||||
def _next(self, stop: threading.Event) -> protocol.Request | None:
|
||||
"""The next request once its start slot has come; None when stopping."""
|
||||
with self._cv:
|
||||
while not stop.is_set():
|
||||
if not self._queue:
|
||||
self._cv.wait(0.5)
|
||||
continue
|
||||
delay = self._last_start + self._pace - self._monotonic()
|
||||
if delay > 0:
|
||||
self._cv.wait(delay)
|
||||
continue
|
||||
self._last_start = self._monotonic()
|
||||
req = self._queue.popleft()
|
||||
self.busy = req.rid
|
||||
return req
|
||||
return None
|
||||
|
||||
def _run_one(self, req: protocol.Request) -> dict[str, Any]:
|
||||
try:
|
||||
return self._handle(req)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# The traceback goes to the journal only; kuhy sees one line.
|
||||
_logger.exception("request %s (%s) crashed", req.rid, req.kind)
|
||||
return protocol.failure(
|
||||
"worker_error",
|
||||
"The coach hit an internal error; see the relay's journal.",
|
||||
at=self._clock(),
|
||||
)
|
||||
|
||||
def _tell(self, busy: str | None) -> None:
|
||||
for listener in self.on_busy:
|
||||
listener(busy)
|
||||
|
||||
def run(self, stop: threading.Event) -> None:
|
||||
"""Run requests until `stop`; a request already running finishes."""
|
||||
while (req := self._next(stop)) is not None:
|
||||
self._tell(req.rid)
|
||||
result = self._run_one(req)
|
||||
self.done.put(req.rid, req.transport, result)
|
||||
with self._cv:
|
||||
self.busy = None
|
||||
self._tell(None)
|
||||
for listener in self.on_result:
|
||||
listener(req, result)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Wake the worker so it notices its stop event at once."""
|
||||
with self._cv:
|
||||
self._cv.notify_all()
|
||||
205
leetcode_coach/leetcode_coach/relay/phone.py
Normal file
205
leetcode_coach/leetcode_coach/relay/phone.py
Normal file
@ -0,0 +1,205 @@
|
||||
"""The phone's path: RTDB requests in, results/state/chat/heartbeat out.
|
||||
|
||||
Under ``leetcode-guard/coach/`` (DOCS-coach-relay.md 1). Requests arrive over
|
||||
an SSE stream of ``requests`` and, as a backstop, a rescan every
|
||||
`RESCAN_S` and at start-up; both go through `consider`, which answers a rid
|
||||
the done-map knows from the map (re-PUT, never re-run) and queues the rest
|
||||
on the shared lane. Results, ``state`` (a JSON string leaf) and the chat
|
||||
lines are written as they change; the heartbeat every `HEARTBEAT_S` and on
|
||||
every ``rev`` or busy change. ``requests/{rid}`` goes 24 h after its result.
|
||||
|
||||
Firebase being down never stops the relay: every write failure is logged
|
||||
and the next change or beat tries again; the desktop keeps being served.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach.relay import protocol, sse
|
||||
from leetcode_coach.relay.done import DAY_MS
|
||||
from leetcode_coach.relay.rtdb import SERVER_TIME, RtdbError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from leetcode_coach.relay.coach import Coach
|
||||
from leetcode_coach.relay.lane import Lane
|
||||
from leetcode_coach.relay.rtdb import Rtdb
|
||||
from leetcode_coach.relay.session import Session
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
_REQ_DEPTH: Final = 2
|
||||
"""``{rid}/req``: a path two segments deep."""
|
||||
|
||||
HEARTBEAT_S: Final = 30.0
|
||||
RESCAN_S: Final = 60.0
|
||||
|
||||
|
||||
def _pairs(event: sse.Event) -> list[tuple[list[str], object]]:
|
||||
"""``(path segments, value)`` for every node an event set."""
|
||||
base = [s for s in event.path.split("/") if s]
|
||||
if event.kind == "patch" and isinstance(event.data, dict):
|
||||
return [
|
||||
(base + [s for s in str(k).split("/") if s], v)
|
||||
for k, v in event.data.items()
|
||||
]
|
||||
return [(base, event.data)]
|
||||
|
||||
|
||||
class PhoneLink:
|
||||
"""Mirrors the relay into RTDB and feeds RTDB requests to the lane."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
rtdb: Rtdb,
|
||||
lane: Lane,
|
||||
coach: Coach,
|
||||
heartbeat: Callable[[], dict[str, Any]],
|
||||
*,
|
||||
clock_ms: Callable[[], int],
|
||||
heartbeat_s: float = HEARTBEAT_S,
|
||||
rescan_s: float = RESCAN_S,
|
||||
) -> None:
|
||||
"""Bind the database and the relay; call `attach` to start mirroring."""
|
||||
self.rtdb = rtdb
|
||||
self.lane = lane
|
||||
self.coach = coach
|
||||
self._heartbeat = heartbeat
|
||||
self._clock = clock_ms
|
||||
self._beat_every = heartbeat_s
|
||||
self._rescan_every = rescan_s
|
||||
self._write_lock = threading.Lock()
|
||||
self._chat_sid: str | None = None
|
||||
self._sent: set[str] = set()
|
||||
|
||||
def _write(self, what: str, call: Callable[[], None]) -> bool:
|
||||
with self._write_lock:
|
||||
try:
|
||||
call()
|
||||
except RtdbError as exc:
|
||||
_logger.warning("RTDB %s failed: %s", what, exc)
|
||||
return False
|
||||
return True
|
||||
|
||||
def attach(self) -> None:
|
||||
"""Listen to the lane and the backend; then mirror everything once."""
|
||||
self.lane.on_result.append(self.on_result)
|
||||
self.lane.on_busy.append(lambda _busy: self.beat())
|
||||
self.coach.listeners.append(self.on_state)
|
||||
self._chat_sid = None
|
||||
self.on_state(self.coach.state(), self.coach.session)
|
||||
|
||||
def beat(self) -> None:
|
||||
"""PUT the heartbeat (``beat`` as a server timestamp)."""
|
||||
beat = {**self._heartbeat(), "beat": SERVER_TIME}
|
||||
self._write("heartbeat", lambda: self.rtdb.put("heartbeat", beat))
|
||||
|
||||
def put_result(self, rid: str, result: dict[str, Any]) -> None:
|
||||
"""``requests/{rid}/result`` as a JSON string leaf."""
|
||||
text = json.dumps(result, ensure_ascii=False)
|
||||
self._write("result", lambda: self.rtdb.put(f"requests/{rid}/result", text))
|
||||
|
||||
def on_result(self, req: protocol.Request, result: dict[str, Any]) -> None:
|
||||
"""A finished request: answer it if it came from the phone."""
|
||||
if req.transport == "phone":
|
||||
self.put_result(req.rid, result)
|
||||
|
||||
def on_state(self, state: dict[str, Any], session: Session | None) -> None:
|
||||
"""Mirror ``state``, the new chat lines and the heartbeat."""
|
||||
text = json.dumps(state, ensure_ascii=False)
|
||||
self._write("state", lambda: self.rtdb.put("state", text))
|
||||
sid = None if session is None else session.sid
|
||||
if sid != self._chat_sid and self._write(
|
||||
"chat reset", lambda: self.rtdb.delete("chat")
|
||||
):
|
||||
self._chat_sid, self._sent = sid, set()
|
||||
if session is not None and sid == self._chat_sid:
|
||||
new: dict[str, object] = {
|
||||
k: v for k, v in session.chat.items() if k not in self._sent
|
||||
}
|
||||
if new and self._write("chat", lambda: self.rtdb.patch("chat", new)):
|
||||
self._sent |= set(new)
|
||||
self.beat()
|
||||
|
||||
def consider(self, rid: str, node: object) -> None:
|
||||
"""One ``requests/{rid}`` node: answer, queue, or leave it."""
|
||||
if not isinstance(node, dict) or "req" not in node or "result" in node:
|
||||
return
|
||||
if not protocol.RID_RE.fullmatch(rid):
|
||||
_logger.warning("phone request with a bad rid ignored")
|
||||
return
|
||||
if (done := self.lane.done.get(rid)) is not None:
|
||||
self.put_result(rid, done.result)
|
||||
return
|
||||
raw = node["req"]
|
||||
try:
|
||||
req = protocol.parse(
|
||||
{**raw, "rid": rid} if isinstance(raw, dict) else raw, "phone"
|
||||
)
|
||||
except protocol.BadRidError as exc:
|
||||
_logger.warning("phone request %s refused: %s", rid, exc)
|
||||
return
|
||||
except protocol.BadRequestError as exc:
|
||||
_logger.warning("bad phone request %s: %s", rid, exc)
|
||||
result = protocol.failure("bad_request", str(exc), at=self._clock())
|
||||
self.lane.finish(rid, "phone", result)
|
||||
self.put_result(rid, result)
|
||||
return
|
||||
if (immediate := self.lane.submit(req)) is not None:
|
||||
self.put_result(rid, immediate)
|
||||
|
||||
def on_event(self, event: sse.Event) -> None:
|
||||
"""Route one SSE event under ``requests``."""
|
||||
for segs, value in _pairs(event):
|
||||
if not segs and isinstance(value, dict):
|
||||
for rid in sorted(value):
|
||||
self.consider(str(rid), value[rid])
|
||||
elif len(segs) == 1:
|
||||
self.consider(segs[0], value)
|
||||
elif len(segs) == _REQ_DEPTH and segs[1] == "req":
|
||||
self.consider(segs[0], {"req": value})
|
||||
|
||||
def rescan(self) -> None:
|
||||
"""Pick up anything the stream missed; drop answered nodes after 24 h."""
|
||||
try:
|
||||
nodes = self.rtdb.get("requests")
|
||||
except RtdbError as exc:
|
||||
_logger.warning("RTDB rescan failed: %s", exc)
|
||||
return
|
||||
if not isinstance(nodes, dict):
|
||||
return
|
||||
now = self._clock()
|
||||
for rid in sorted(nodes):
|
||||
node = nodes[rid]
|
||||
self.consider(str(rid), node)
|
||||
if not isinstance(node, dict) or "result" not in node:
|
||||
continue
|
||||
done = self.lane.done.get(str(rid))
|
||||
if done is None or now - done.at > DAY_MS:
|
||||
drop = functools.partial(self.rtdb.delete, f"requests/{rid}")
|
||||
self._write("cleanup", drop)
|
||||
|
||||
def listen(self, stop: threading.Event) -> None:
|
||||
"""The SSE loop; a cancelled stream leaves the rescans to carry on."""
|
||||
try:
|
||||
for event in sse.listen(self.rtdb, "requests", stop):
|
||||
self.on_event(event)
|
||||
except sse.StreamCancelledError:
|
||||
_logger.exception("the requests stream was cancelled; rescans only now")
|
||||
|
||||
def tend(self, stop: threading.Event, wait: Callable[[float], object]) -> None:
|
||||
"""Heartbeat every 30 s, rescan every 60 s, until `stop`."""
|
||||
since_scan = self._rescan_every
|
||||
while not stop.is_set():
|
||||
if since_scan >= self._rescan_every:
|
||||
self.rescan()
|
||||
since_scan = 0.0
|
||||
self.beat()
|
||||
wait(self._beat_every)
|
||||
since_scan += self._beat_every
|
||||
74
leetcode_coach/leetcode_coach/relay/probe.py
Normal file
74
leetcode_coach/leetcode_coach/relay/probe.py
Normal file
@ -0,0 +1,74 @@
|
||||
"""Step 4: has LeetCode seen an Accepted for this problem since it started?
|
||||
|
||||
The harvester's own read-only probe (``fetch_recent_ac`` with the client
|
||||
``_settings.build_client`` builds), at most once a minute: LeetCode answers
|
||||
a rate limit exactly like a dead session, so polling harder manufactures
|
||||
the failure. Three answers, never two: ``accepted``, ``not_yet``, and
|
||||
``unknown`` when the probe is unusable -- "could not check" is never "not
|
||||
solved" (CLAUDE.md). The probe mints nothing; crediting stays the
|
||||
harvester's job. ``timestamp`` is unix SECONDS, ``started`` is ms.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from leetcode_guard._settings import build_client
|
||||
from leetcode_guard._submissions import SolveProbe, fetch_recent_ac
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
EVERY_S: Final = 60.0
|
||||
ACCEPTED: Final = "accepted"
|
||||
NOT_YET: Final = "not_yet"
|
||||
UNKNOWN: Final = "unknown"
|
||||
|
||||
|
||||
def live_probe() -> SolveProbe:
|
||||
"""One read of kuhy's recent accepted submissions."""
|
||||
client = build_client()
|
||||
return fetch_recent_ac(client.post, client.username)
|
||||
|
||||
|
||||
class AcceptedProbe:
|
||||
"""Rate-limited, three-valued Accepted check."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
probe: Callable[[], SolveProbe] = live_probe,
|
||||
monotonic: Callable[[], float] = time.monotonic,
|
||||
every_s: float = EVERY_S,
|
||||
) -> None:
|
||||
"""Bind the probe and the clock that spaces it."""
|
||||
self._probe = probe
|
||||
self._monotonic = monotonic
|
||||
self._every = every_s
|
||||
self._last: float | None = None
|
||||
|
||||
def due(self) -> bool:
|
||||
"""Whether a minute has passed since the last probe."""
|
||||
last = self._last
|
||||
return last is None or self._monotonic() - last >= self._every
|
||||
|
||||
def check(self, slug: str, started_ms: int) -> str:
|
||||
"""``accepted`` / ``not_yet`` / ``unknown`` for `slug` since `started_ms`."""
|
||||
self._last = self._monotonic()
|
||||
try:
|
||||
answer = self._probe()
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# Anything the probe raises is a probe we cannot believe.
|
||||
_logger.exception("the Accepted probe failed")
|
||||
return UNKNOWN
|
||||
if not answer.usable:
|
||||
_logger.warning("Accepted probe unusable: %s", answer.reason)
|
||||
return UNKNOWN
|
||||
seen = any(
|
||||
s.title_slug == slug and s.timestamp * 1000 >= started_ms
|
||||
for s in answer.submissions
|
||||
)
|
||||
return ACCEPTED if seen else NOT_YET
|
||||
176
leetcode_coach/leetcode_coach/relay/protocol.py
Normal file
176
leetcode_coach/leetcode_coach/relay/protocol.py
Normal file
@ -0,0 +1,176 @@
|
||||
"""The seven request kinds and their limits (DOCS-coach-relay.md 2 and 4).
|
||||
|
||||
`parse` is the only door into the relay: a request is checked here, field by
|
||||
field, before anything builds a path or reaches a model. Unknown fields are
|
||||
dropped, a kind outside the seven is never run, and an id is regex-checked
|
||||
first. A bad ``rid`` raises `BadRidError` (it gets no result at all: there is
|
||||
nowhere safe to put one); anything else wrong is `BadRequestError`, answered
|
||||
``bad_request``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Final, NoReturn
|
||||
|
||||
KINDS: Final = (
|
||||
"session_start",
|
||||
"session_end",
|
||||
"chat",
|
||||
"grade_step",
|
||||
"hint",
|
||||
"reveal",
|
||||
"run_code",
|
||||
)
|
||||
RID_RE: Final = re.compile(r"[0-9A-HJKMNP-TV-Z]{26}")
|
||||
"""A ULID: 26 Crockford base-32 characters (no I, L, O, U)."""
|
||||
SID_RE: Final = re.compile(r"\d{8}-\d{6}-[0-9a-f]{4}")
|
||||
DEV_RE: Final = re.compile(
|
||||
r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
|
||||
)
|
||||
SLUG_RE: Final = re.compile(r"[a-z0-9]+(-[a-z0-9]+)*")
|
||||
MAX_SLUG: Final = 100
|
||||
MAX_TEXT_BYTES: Final = 8192
|
||||
MAX_REQ_BYTES: Final = 16 * 1024
|
||||
MAX_PICKS: Final = 30
|
||||
MAX_PICK_CHARS: Final = 64
|
||||
GRADED_STEPS: Final = (1, 2, 3, 5)
|
||||
TOOLBOX_STEP: Final = 3
|
||||
ALL_STEPS: Final = range(1, 6)
|
||||
RETRYABLE: Final = frozenset({"rate_limited", "model_error"})
|
||||
|
||||
|
||||
class BadRidError(ValueError):
|
||||
"""The rid is not a ULID: the request gets no result anywhere."""
|
||||
|
||||
|
||||
class BadRequestError(ValueError):
|
||||
"""Anything else wrong with a request; answered ``bad_request``."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Request:
|
||||
"""One validated request: only the fields its kind takes survive."""
|
||||
|
||||
rid: str
|
||||
kind: str
|
||||
dev: str
|
||||
body: dict[str, Any]
|
||||
transport: str
|
||||
"""``desktop`` (loopback API) or ``phone`` (RTDB)."""
|
||||
|
||||
|
||||
def _bad(message: str) -> NoReturn:
|
||||
raise BadRequestError(message)
|
||||
|
||||
|
||||
def _sid(body: dict[str, Any]) -> dict[str, Any]:
|
||||
sid = body.get("session")
|
||||
if not isinstance(sid, str) or not SID_RE.fullmatch(sid):
|
||||
_bad("session must be a session id")
|
||||
return {"session": sid}
|
||||
|
||||
|
||||
def _text(body: dict[str, Any], key: str) -> str:
|
||||
value = body.get(key)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
_bad(f"{key} must be a non-empty string")
|
||||
if len(value.encode()) > MAX_TEXT_BYTES:
|
||||
_bad(f"{key} is over {MAX_TEXT_BYTES} bytes")
|
||||
return value
|
||||
|
||||
|
||||
def _picks(body: dict[str, Any]) -> list[str]:
|
||||
picks = body.get("picks")
|
||||
if not isinstance(picks, list) or len(picks) > MAX_PICKS:
|
||||
_bad(f"picks must be a list of at most {MAX_PICKS} names")
|
||||
if not all(isinstance(p, str) and 0 < len(p) <= MAX_PICK_CHARS for p in picks):
|
||||
_bad(f"every pick must be a name of at most {MAX_PICK_CHARS} chars")
|
||||
return picks
|
||||
|
||||
|
||||
def _grade(body: dict[str, Any]) -> dict[str, Any]:
|
||||
step = body.get("step")
|
||||
if isinstance(step, bool) or not isinstance(step, int) or step not in ALL_STEPS:
|
||||
_bad("step must be 1 to 5")
|
||||
out: dict[str, Any] = {**_sid(body), "step": step}
|
||||
if "force" in body:
|
||||
if body["force"] is not True:
|
||||
_bad("force must be true when given")
|
||||
return {**out, "force": True}
|
||||
if step not in GRADED_STEPS:
|
||||
_bad("step 4 is never graded: it passes on a LeetCode Accepted")
|
||||
if step == TOOLBOX_STEP: # it takes picks; the others take text
|
||||
return {**out, "picks": _picks(body)}
|
||||
return {**out, "answer": _text(body, "answer")}
|
||||
|
||||
|
||||
def _body(kind: str, body: dict[str, Any]) -> dict[str, Any]:
|
||||
if kind == "session_start":
|
||||
slug = body.get("slug")
|
||||
if not isinstance(slug, str) or len(slug) > MAX_SLUG:
|
||||
_bad("slug must be a LeetCode slug")
|
||||
if not SLUG_RE.fullmatch(slug):
|
||||
_bad("slug must be a LeetCode slug")
|
||||
return {"slug": slug}
|
||||
if kind == "chat":
|
||||
return {**_sid(body), "text": _text(body, "text")}
|
||||
if kind == "run_code":
|
||||
return {**_sid(body), "code": _text(body, "code")}
|
||||
if kind == "grade_step":
|
||||
return _grade(body)
|
||||
return _sid(body)
|
||||
|
||||
|
||||
def _no_rid(message: str) -> NoReturn:
|
||||
raise BadRidError(message)
|
||||
|
||||
|
||||
def parse(raw: object, transport: str) -> Request:
|
||||
"""The validated request in `raw` (a decoded ``req`` object).
|
||||
|
||||
Raises:
|
||||
BadRidError: when there is no valid rid to answer under.
|
||||
BadRequestError: for every other problem.
|
||||
"""
|
||||
rid = raw.get("rid") if isinstance(raw, dict) else None
|
||||
if not isinstance(raw, dict) or not isinstance(rid, str):
|
||||
_no_rid("a request needs a rid")
|
||||
if not RID_RE.fullmatch(rid):
|
||||
_no_rid("the rid is not a ULID")
|
||||
if len(json.dumps(raw, ensure_ascii=False).encode()) > MAX_REQ_BYTES:
|
||||
_bad(f"the request is over {MAX_REQ_BYTES} bytes")
|
||||
kind, dev, body = raw.get("kind"), raw.get("dev"), raw.get("body")
|
||||
if kind not in KINDS:
|
||||
_bad("unknown request kind")
|
||||
if not isinstance(dev, str) or not DEV_RE.fullmatch(dev):
|
||||
_bad("dev must be the app's install uuid")
|
||||
if not isinstance(body, dict):
|
||||
_bad("body must be an object")
|
||||
return Request(rid, str(kind), dev, _body(str(kind), body), transport)
|
||||
|
||||
|
||||
def ok(data: dict[str, Any], *, at: int, state_rev: int) -> dict[str, Any]:
|
||||
"""A success result."""
|
||||
return {"ok": True, "at": at, "state_rev": state_rev, "data": data}
|
||||
|
||||
|
||||
def failure(
|
||||
code: str, error: str, *, at: int, retry: bool = False, **extra: object
|
||||
) -> dict[str, Any]:
|
||||
"""An error result; `error` is shown to kuhy as it stands."""
|
||||
return {
|
||||
"ok": False,
|
||||
"at": at,
|
||||
"code": code,
|
||||
"error": error,
|
||||
"retry": retry,
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def ok_retryable(code: str) -> bool:
|
||||
"""Whether an error with `code` is one kuhy resends (as a NEW rid)."""
|
||||
return code in RETRYABLE
|
||||
149
leetcode_coach/leetcode_coach/relay/rtdb.py
Normal file
149
leetcode_coach/leetcode_coach/relay/rtdb.py
Normal file
@ -0,0 +1,149 @@
|
||||
"""A minimal synchronous RTDB REST client: GET/PUT/PATCH/DELETE and SSE streams.
|
||||
|
||||
Ported from tutor's async ``tutor/relay/rtdb.py`` onto ``requests`` (this repo's
|
||||
one network dependency; the relay runs on threads, not asyncio). Every call
|
||||
authenticates with ``?auth=<idToken>`` (RTDB REST has no header form), so
|
||||
nothing here ever formats a URL or a ``requests`` error into a message: errors
|
||||
name the method, the path and the status only. Writes ask for
|
||||
``print=silent``, so RTDB does not echo every body back (`_snapshot_rtdb`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from urllib.parse import quote
|
||||
|
||||
import requests
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Generator
|
||||
|
||||
ROOT: Final = "leetcode-guard/coach"
|
||||
SERVER_TIME: Final = {".sv": "timestamp"}
|
||||
_CONNECT_S: Final = 15.0
|
||||
_STREAM_READ_S: Final = 75.0
|
||||
"""RTDB sends a keep-alive every 30 s, so 75 s of silence is a dead stream."""
|
||||
|
||||
# urllib3 logs every request line, query string (the token) included, at DEBUG.
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
|
||||
|
||||
class RtdbError(RuntimeError):
|
||||
"""A request failed; the message never contains the URL or the token."""
|
||||
|
||||
|
||||
def parse(lines: list[str]) -> tuple[str, str]:
|
||||
"""``(event, data)`` of one SSE block (its lines, blank line excluded)."""
|
||||
event, data = "message", []
|
||||
for line in lines:
|
||||
field, _, value = line.partition(":")
|
||||
value = value.removeprefix(" ")
|
||||
if field == "event":
|
||||
event = value
|
||||
elif field == "data":
|
||||
data.append(value)
|
||||
return event, "\n".join(data)
|
||||
|
||||
|
||||
class Rtdb:
|
||||
"""REST calls under ``/leetcode-guard/coach``; ``id_token`` yields a fresh token."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
database_url: str,
|
||||
id_token: Callable[[], str],
|
||||
*,
|
||||
session: requests.Session | None = None,
|
||||
timeout_s: float = 30.0,
|
||||
) -> None:
|
||||
"""Bind the database; ``session`` is injectable for tests."""
|
||||
self._base = database_url.rstrip("/")
|
||||
self._token = id_token
|
||||
self._session = session or requests.Session()
|
||||
self._timeout = timeout_s
|
||||
|
||||
def url(self, path: str) -> str:
|
||||
"""The REST URL of ``path`` (relative to the coach root), no token."""
|
||||
rel = "/".join(quote(p, safe="") for p in path.strip("/").split("/") if p)
|
||||
return f"{self._base}/{ROOT}/{rel}.json" if rel else f"{self._base}/{ROOT}.json"
|
||||
|
||||
def _call(
|
||||
self, method: str, path: str, value: object = None, *, body: bool = False
|
||||
) -> requests.Response:
|
||||
params = {"auth": self._token()}
|
||||
if body:
|
||||
params["print"] = "silent"
|
||||
data = json.dumps(value).encode() if body else None
|
||||
try:
|
||||
response = self._session.request(
|
||||
method, self.url(path), params=params, data=data, timeout=self._timeout
|
||||
)
|
||||
except requests.RequestException as err:
|
||||
msg = f"{method} {path}: {type(err).__name__}"
|
||||
raise RtdbError(msg) from None
|
||||
if not response.ok:
|
||||
msg = f"{method} {path}: HTTP {response.status_code}"
|
||||
raise RtdbError(msg)
|
||||
return response
|
||||
|
||||
def get(self, path: str) -> object:
|
||||
"""The value at ``path`` (``None`` when absent)."""
|
||||
response = self._call("GET", path)
|
||||
try:
|
||||
value: object = response.json()
|
||||
except ValueError:
|
||||
msg = f"GET {path}: not JSON"
|
||||
raise RtdbError(msg) from None
|
||||
return value
|
||||
|
||||
def put(self, path: str, value: object) -> None:
|
||||
"""Replace ``path`` with ``value`` (a str is stored as a string leaf)."""
|
||||
self._call("PUT", path, value, body=True)
|
||||
|
||||
def patch(self, path: str, value: dict[str, object]) -> None:
|
||||
"""Merge ``value``'s children (keys may be nested ``a/b`` paths)."""
|
||||
self._call("PATCH", path, value, body=True)
|
||||
|
||||
def delete(self, path: str) -> None:
|
||||
"""Remove ``path``."""
|
||||
self._call("DELETE", path)
|
||||
|
||||
def stream(self, path: str) -> Generator[tuple[str, str]]:
|
||||
"""The ``(event, data)`` blocks of ONE SSE connection to ``path``.
|
||||
|
||||
Ends at EOF; the response is closed however the iteration stops.
|
||||
|
||||
Raises:
|
||||
RtdbError: The connection could not open, or it dropped mid-read.
|
||||
"""
|
||||
try:
|
||||
response = self._session.get(
|
||||
self.url(path),
|
||||
params={"auth": self._token()},
|
||||
headers={"Accept": "text/event-stream"},
|
||||
stream=True,
|
||||
timeout=(_CONNECT_S, _STREAM_READ_S),
|
||||
)
|
||||
except requests.RequestException as err:
|
||||
msg = f"STREAM {path}: {type(err).__name__}"
|
||||
raise RtdbError(msg) from None
|
||||
try:
|
||||
if not response.ok:
|
||||
msg = f"STREAM {path}: HTTP {response.status_code}"
|
||||
raise RtdbError(msg)
|
||||
block: list[str] = []
|
||||
# chunk_size=1: a non-chunked stream would otherwise hold a small
|
||||
# event back until 512 bytes arrived (SSE events are tiny).
|
||||
for raw in response.iter_lines(chunk_size=1):
|
||||
if raw:
|
||||
block.append(raw.decode("utf-8", errors="replace"))
|
||||
elif block:
|
||||
yield parse(block)
|
||||
block = []
|
||||
except requests.RequestException as err:
|
||||
msg = f"STREAM {path}: {type(err).__name__}"
|
||||
raise RtdbError(msg) from None
|
||||
finally:
|
||||
response.close()
|
||||
196
leetcode_coach/leetcode_coach/relay/runcode.py
Normal file
196
leetcode_coach/leetcode_coach/relay/runcode.py
Normal file
@ -0,0 +1,196 @@
|
||||
"""run_code: kuhy's JavaScript against the statement's examples (decision 8).
|
||||
|
||||
The contract (DOCS-coach-relay.md section 2) allows exactly one subprocess
|
||||
shape here: ``node`` with a fixed argv, never a shell. kuhy's code is embedded
|
||||
in a generated harness as a JSON string literal and compiled with
|
||||
``new Function``, so a syntax error is told apart from a crash; the examples
|
||||
are JSON literals this module generates from the statement. The process runs
|
||||
in a fresh temporary directory under node's permission model: it may read
|
||||
that directory and nothing else, spawns no child, opens no worker.
|
||||
|
||||
The answer carries printed values only, never node's stderr or a stack trace:
|
||||
a thrown error becomes ``error: <ClassName>``. The harness silences
|
||||
``console`` and ``process.stdout.write`` before kuhy's code loads and marks its
|
||||
own result lines with a per-run random token, so whatever kuhy's code prints
|
||||
cannot pose as a result. Output is cut at ``MAX_OUTPUT`` bytes before it is
|
||||
read, and one value is shown in at most ``MAX_SHOWN`` characters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach.relay import examples
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
NODE_ENV: Final = "LEETCODE_NODE_BIN"
|
||||
DEFAULT_NODE: Final = "/usr/bin/node"
|
||||
TIMEOUT_S: Final = 5.0
|
||||
MAX_OUTPUT: Final = 64 * 1024
|
||||
MAX_SHOWN: Final = 200
|
||||
NO_RESULT: Final = "no result"
|
||||
_FIELDS: Final = 4 # mark, n, ok|err, value
|
||||
_ERROR_NAME: Final = re.compile(r"[A-Za-z]{1,40}")
|
||||
|
||||
_HARNESS: Final = """const __mark = "{mark}";
|
||||
const __write = process.stdout.write.bind(process.stdout);
|
||||
for (const k of ["log", "error", "warn", "info", "debug", "trace", "dir", "table"]) {{
|
||||
console[k] = () => {{}};
|
||||
}}
|
||||
process.stdout.write = () => true;
|
||||
process.stderr.write = () => true;
|
||||
const __say = (line) => __write(__mark + " " + line + "\\n");
|
||||
const __name = (e) =>
|
||||
e && typeof e.name === "string" && /^[A-Za-z]{{1,40}}$/.test(e.name)
|
||||
? e.name
|
||||
: "Error";
|
||||
const __examples = {examples};
|
||||
let __load;
|
||||
try {{
|
||||
__load = new Function(
|
||||
{code} + "\\n;return typeof {name} === 'function' ? {name} : undefined;"
|
||||
);
|
||||
}} catch (e) {{
|
||||
__say("syntax");
|
||||
process.exit(0);
|
||||
}}
|
||||
let __fn;
|
||||
let __failed = null;
|
||||
try {{
|
||||
__fn = __load();
|
||||
}} catch (e) {{
|
||||
__failed = __name(e);
|
||||
}}
|
||||
if (__failed === null && typeof __fn !== "function") __failed = "ReferenceError";
|
||||
for (const ex of __examples) {{
|
||||
if (__failed !== null) {{
|
||||
__say(ex.n + " err " + __failed);
|
||||
continue;
|
||||
}}
|
||||
try {{
|
||||
const r = __fn(...ex.args);
|
||||
__say(ex.n + " ok " + (r === undefined ? "undefined" : JSON.stringify(r)));
|
||||
}} catch (e) {{
|
||||
__say(ex.n + " err " + __name(e));
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
def node_bin() -> str:
|
||||
"""``LEETCODE_NODE_BIN``, else ``node`` from PATH, else ``/usr/bin/node``."""
|
||||
return os.environ.get(NODE_ENV) or shutil.which("node") or DEFAULT_NODE
|
||||
|
||||
|
||||
def _canonical(value: object) -> str:
|
||||
return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
|
||||
def _shown(text: str) -> str:
|
||||
return text if len(text) <= MAX_SHOWN else text[:MAX_SHOWN] + "..."
|
||||
|
||||
|
||||
def harness(code: str, name: str, found: list[examples.Example], mark: str) -> str:
|
||||
"""The generated ``run.js``: every interpolated value is a JSON literal."""
|
||||
cases = [{"n": ex.n, "args": list(ex.args)} for ex in found]
|
||||
return _HARNESS.format(
|
||||
mark=mark,
|
||||
examples=json.dumps(cases),
|
||||
code=json.dumps(code),
|
||||
name=name,
|
||||
)
|
||||
|
||||
|
||||
def _results(stdout: bytes, mark: str) -> dict[int, tuple[str, str]] | None:
|
||||
"""``{n: (ok|err, value)}`` from the marked lines; None on a syntax error."""
|
||||
text = stdout[:MAX_OUTPUT].decode("utf-8", errors="replace")
|
||||
found: dict[int, tuple[str, str]] = {}
|
||||
for line in text.splitlines():
|
||||
parts = line.split(" ", 3)
|
||||
if parts[0] != mark:
|
||||
continue
|
||||
if parts[1:] == ["syntax"]:
|
||||
return None
|
||||
if len(parts) == _FIELDS and parts[1].isdigit() and parts[2] in {"ok", "err"}:
|
||||
found[int(parts[1])] = (parts[2], parts[3])
|
||||
return found
|
||||
|
||||
|
||||
def _row(ex: examples.Example, result: tuple[str, str] | None) -> dict[str, Any]:
|
||||
expected = _canonical(ex.expected)
|
||||
row: dict[str, Any] = {"n": ex.n, "ok": False, "expected": _shown(expected)}
|
||||
if result is None:
|
||||
return {**row, "got": NO_RESULT}
|
||||
kind, value = result
|
||||
if kind == "err":
|
||||
name = value if _ERROR_NAME.fullmatch(value) else "Error"
|
||||
return {**row, "got": f"error: {name}"}
|
||||
try:
|
||||
got = _canonical(json.loads(value))
|
||||
except ValueError:
|
||||
_logger.warning("example %d printed a value that is not JSON", ex.n)
|
||||
return {**row, "got": _shown(value)}
|
||||
return {**row, "ok": got == expected, "got": _shown(got)}
|
||||
|
||||
|
||||
def _failed(error: str) -> dict[str, Any]:
|
||||
return {"examples": [], "error": error}
|
||||
|
||||
|
||||
def run(
|
||||
code: str,
|
||||
statement: str,
|
||||
*,
|
||||
node: str | None = None,
|
||||
timeout_s: float = TIMEOUT_S,
|
||||
runner: Callable[..., subprocess.CompletedProcess[bytes]] = subprocess.run,
|
||||
) -> dict[str, Any]:
|
||||
"""Run ``code`` on every parsed example; the contract's ``run_code`` data."""
|
||||
name, found = examples.parse(statement)
|
||||
if name is None or not found:
|
||||
return _failed("no_examples")
|
||||
mark = secrets.token_hex(16)
|
||||
with tempfile.TemporaryDirectory(prefix="leetcode-run-") as tmp:
|
||||
script = Path(tmp) / "run.js"
|
||||
script.write_text(harness(code, name, found, mark), encoding="utf-8")
|
||||
argv = [
|
||||
node or node_bin(),
|
||||
"--permission",
|
||||
f"--allow-fs-read={tmp}",
|
||||
"--max-old-space-size=128",
|
||||
str(script),
|
||||
]
|
||||
try:
|
||||
proc = runner(
|
||||
argv,
|
||||
cwd=tmp,
|
||||
env={"PATH": "/usr/bin:/bin"},
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
timeout=timeout_s,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
_logger.warning("run_code: node ran past %.0f s and was killed", timeout_s)
|
||||
return _failed("timeout")
|
||||
except OSError as exc:
|
||||
_logger.warning("run_code: node could not start: %s", exc)
|
||||
return _failed("node_missing")
|
||||
results = _results(proc.stdout, mark)
|
||||
if results is None:
|
||||
return _failed("syntax")
|
||||
rows = [_row(ex, results.get(ex.n)) for ex in found]
|
||||
return {"examples": rows, "error": None}
|
||||
167
leetcode_coach/leetcode_coach/relay/session.py
Normal file
167
leetcode_coach/leetcode_coach/relay/session.py
Normal file
@ -0,0 +1,167 @@
|
||||
"""The live coaching session: what the log cannot hold, and its pointer file.
|
||||
|
||||
Steps, grades, rung and reveal are derived from the session log every time
|
||||
(`steps`); a `Session` adds what only the relay knows -- the claude session
|
||||
id, the chat lines in display order, notes waiting for the coach's next turn
|
||||
and the Accepted probe. ``live.json`` points at the live session (sid, log,
|
||||
claude id, rev) so a restarted relay rebuilds it from the log: chat lines
|
||||
from the ``chat`` and ``grade`` events, the rest re-derived.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from leetcode_coach import session_log
|
||||
from leetcode_coach.relay.steps import NAMES
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from leetcode_coach.local_coach import LocalCoach
|
||||
from leetcode_coach.phase import PhaseGate
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
LIVE_FIELDS: Final = (
|
||||
"sid",
|
||||
"slug",
|
||||
"title",
|
||||
"url",
|
||||
"cluster",
|
||||
"started",
|
||||
"transport",
|
||||
"dev",
|
||||
"claude_id",
|
||||
"local",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Session: # pylint: disable=too-many-instance-attributes
|
||||
"""One problem's coaching session."""
|
||||
|
||||
sid: str
|
||||
slug: str
|
||||
title: str
|
||||
url: str
|
||||
cluster: str | None
|
||||
log: Path
|
||||
started: int
|
||||
transport: str
|
||||
dev: str
|
||||
gate: PhaseGate
|
||||
statement: str | None = None
|
||||
claude_id: str | None = None
|
||||
local: bool = False
|
||||
ended: int | None = None
|
||||
end_reason: str | None = None
|
||||
chat: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
notes: list[str] = field(default_factory=list)
|
||||
history: list[dict[str, str]] = field(default_factory=list)
|
||||
local_coach: LocalCoach | None = None
|
||||
probe: str | None = None
|
||||
probed_at: int | None = None
|
||||
|
||||
def events(self) -> list[dict[str, Any]]:
|
||||
"""This session's log events (from its ``session_start`` on)."""
|
||||
return session_log.current(session_log.read(self.log))
|
||||
|
||||
def add(self, role: str, kind: str, text: str, at: int) -> str:
|
||||
"""Append one chat line (empty fields dropped); its ``m0001`` key."""
|
||||
key = f"m{len(self.chat) + 1:04d}"
|
||||
line = {"role": role, "kind": kind, "text": text, "at": at}
|
||||
self.chat[key] = {k: v for k, v in line.items() if v not in ("", None)}
|
||||
return key
|
||||
|
||||
def log_event(self, event: str, **fields: object) -> None:
|
||||
"""Append `event` to the log with this session's sid and cluster."""
|
||||
extra = {"cluster": self.cluster} if self.cluster else {}
|
||||
session_log.append(self.log, event, sid=self.sid, **extra, **fields)
|
||||
|
||||
def info(self) -> dict[str, Any]:
|
||||
"""The ``session`` block of the published state."""
|
||||
return {
|
||||
"id": self.sid,
|
||||
"slug": self.slug,
|
||||
"title": self.title,
|
||||
"cluster": self.cluster,
|
||||
"url": self.url,
|
||||
"started": self.started,
|
||||
"ended": self.ended,
|
||||
"end_reason": self.end_reason,
|
||||
}
|
||||
|
||||
|
||||
def save(path: Path, session: Session, rev: int) -> None:
|
||||
"""Write the live pointer (atomically: a torn pointer loses the session)."""
|
||||
record = {k: getattr(session, k) for k in LIVE_FIELDS}
|
||||
record |= {"log": str(session.log), "rev": rev}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(record), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _ms(event: dict[str, Any]) -> int:
|
||||
at = event.get("at")
|
||||
if isinstance(at, int):
|
||||
return at
|
||||
try:
|
||||
return int(datetime.fromisoformat(str(event.get("ts"))).timestamp() * 1000)
|
||||
except ValueError as exc:
|
||||
_logger.warning("event without a usable time: %s", exc)
|
||||
return 0
|
||||
|
||||
|
||||
def grade_line(event: dict[str, Any]) -> tuple[str, str]:
|
||||
"""The ``system`` chat line a step's grade shows as."""
|
||||
step = int(event["step"])
|
||||
name = NAMES.get(step, "step")
|
||||
if event.get("via") == "accepted":
|
||||
return "accepted", f"Step 4 ({name}): LeetCode shows an Accepted. Step 5 open."
|
||||
if event.get("forced") is True:
|
||||
tries = event.get("attempt")
|
||||
return "grade", f"Step {step} ({name}): continued after {tries} tries."
|
||||
head = f"Step {step} ({name}): {event.get('grade')} on try {event.get('attempt')}."
|
||||
feedback = str(event.get("feedback") or "")
|
||||
return "grade", f"{head}\n{feedback}".strip()
|
||||
|
||||
|
||||
def restore(
|
||||
path: Path, gate_for: Callable[[Path, str], PhaseGate]
|
||||
) -> tuple[Session, int] | None:
|
||||
"""The session ``live.json`` points at, rebuilt from its log; None if gone."""
|
||||
if not path.is_file():
|
||||
return None
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
log = Path(raw["log"])
|
||||
fields = {k: raw[k] for k in LIVE_FIELDS}
|
||||
rev = int(raw.get("rev", 0))
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
_logger.warning("live session pointer %s is unusable: %s", path, exc)
|
||||
return None
|
||||
session = Session(**fields, log=log, gate=gate_for(log, raw["slug"]))
|
||||
events = session.events()
|
||||
if not events or events[0].get("sid") != session.sid:
|
||||
_logger.warning("live session %s is not the log's latest", session.sid)
|
||||
return None
|
||||
for event in events:
|
||||
if event["event"] == "chat":
|
||||
role, kind = str(event.get("role")), str(event.get("kind", "chat"))
|
||||
session.add(role, kind, str(event.get("text", "")), _ms(event))
|
||||
who = "user" if role == "kuhy" else "assistant"
|
||||
session.history.append({"role": who, "content": str(event["text"])})
|
||||
elif event["event"] == "grade":
|
||||
kind, text = grade_line(event)
|
||||
session.add("system", kind, text, _ms(event))
|
||||
elif event["event"] == "session_end":
|
||||
session.ended = _ms(event)
|
||||
session.end_reason = str(event.get("reason"))
|
||||
return session, rev
|
||||
120
leetcode_coach/leetcode_coach/relay/sse.py
Normal file
120
leetcode_coach/leetcode_coach/relay/sse.py
Normal file
@ -0,0 +1,120 @@
|
||||
"""RTDB streaming: Server-Sent Events, reconnected until told to stop.
|
||||
|
||||
tutor's ``sse.py`` on threads. Each (re)connect starts with a ``put`` of the
|
||||
whole node at ``/``, so a consumer that applies ``put``/``patch`` events is
|
||||
correct again after any drop. ``auth_revoked`` (the id token expired)
|
||||
reconnects at once with a fresh token; a network error, an HTTP error, a bad
|
||||
payload or a connection that closes before saying anything backs off 1-30 s;
|
||||
``cancel`` (the rules now refuse the path) is fatal. `stop` is checked
|
||||
between events: a quiet stream notices it at its next keep-alive (<= 30 s).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from leetcode_coach.relay.rtdb import RtdbError, parse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Generator, Iterator
|
||||
import threading
|
||||
|
||||
__all__ = ["BACKOFF_S", "Event", "StreamCancelledError", "listen", "parse"]
|
||||
|
||||
BACKOFF_S: Final = (1.0, 2.0, 5.0, 10.0, 30.0)
|
||||
_DATA_EVENTS: Final = frozenset({"put", "patch"})
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Event:
|
||||
"""One ``put``/``patch`` event: where it applies and the new data."""
|
||||
|
||||
kind: str
|
||||
path: str
|
||||
data: object
|
||||
|
||||
|
||||
class StreamCancelledError(RtdbError):
|
||||
"""RTDB cancelled the stream (rules refuse the path): not retried."""
|
||||
|
||||
|
||||
class Streams(Protocol):
|
||||
"""What `listen` needs of `rtdb.Rtdb`."""
|
||||
|
||||
def stream(self, path: str) -> Generator[tuple[str, str]]:
|
||||
"""The raw blocks of one connection."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Health:
|
||||
"""Whether a connection proved itself (an event, or a clean revocation)."""
|
||||
|
||||
good: bool = False
|
||||
|
||||
|
||||
def _event(kind: str, raw: str) -> Event:
|
||||
payload = json.loads(raw)
|
||||
if not isinstance(payload, dict):
|
||||
msg = f"a {kind} payload is not an object"
|
||||
raise TypeError(msg)
|
||||
return Event(kind, str(payload.get("path", "/")), payload.get("data"))
|
||||
|
||||
|
||||
def _connection(
|
||||
rtdb: Streams, path: str, stop: threading.Event, health: _Health
|
||||
) -> Iterator[Event]:
|
||||
"""Events of a single connection; returns on EOF, revocation or `stop`."""
|
||||
with closing(rtdb.stream(path)) as blocks:
|
||||
for kind, raw in blocks:
|
||||
if kind in _DATA_EVENTS:
|
||||
event = _event(kind, raw)
|
||||
health.good = True
|
||||
yield event
|
||||
if stop.is_set():
|
||||
return
|
||||
elif kind == "auth_revoked":
|
||||
_logger.info("stream %s: token expired, reconnecting", path)
|
||||
health.good = True
|
||||
return
|
||||
elif kind == "cancel":
|
||||
msg = f"stream {path} cancelled by the server"
|
||||
raise StreamCancelledError(msg)
|
||||
|
||||
|
||||
def listen(
|
||||
rtdb: Streams,
|
||||
path: str,
|
||||
stop: threading.Event,
|
||||
sleep: Callable[[float], object] | None = None,
|
||||
) -> Iterator[Event]:
|
||||
"""Every event at ``path`` across reconnects, until `stop` is set."""
|
||||
pause = stop.wait if sleep is None else sleep
|
||||
failures = 0
|
||||
while not stop.is_set():
|
||||
health = _Health()
|
||||
reason = "closed before any event"
|
||||
try:
|
||||
yield from _connection(rtdb, path, stop, health)
|
||||
except StreamCancelledError:
|
||||
raise
|
||||
except (RtdbError, ValueError, TypeError) as err:
|
||||
# ValueError: a payload that is not JSON; TypeError: not an object.
|
||||
reason = str(err)
|
||||
_logger.warning("stream %s dropped: %s", path, reason)
|
||||
else:
|
||||
if health.good:
|
||||
failures = 0
|
||||
continue
|
||||
if health.good:
|
||||
failures = 0
|
||||
if stop.is_set():
|
||||
return
|
||||
delay = BACKOFF_S[min(failures, len(BACKOFF_S) - 1)]
|
||||
failures += 1
|
||||
_logger.warning("stream %s: %s; retry in %.0fs", path, reason, delay)
|
||||
pause(delay)
|
||||
129
leetcode_coach/leetcode_coach/relay/steps.py
Normal file
129
leetcode_coach/leetcode_coach/relay/steps.py
Normal file
@ -0,0 +1,129 @@
|
||||
"""The five-step state machine, derived from the session log (contract 2-3).
|
||||
|
||||
Nothing about a step is held in memory: its status, tries and last grade are
|
||||
read back from this session's events every time, so a relay restart, the CLI
|
||||
writing into the same file, and the app all agree. Each step goes
|
||||
``locked -> open -> passed | forced``; passing or forcing step n opens n+1.
|
||||
A ``partial`` / ``fail`` keeps the step open and counts a try; after
|
||||
`FORCE_AFTER` tries without a pass kuhy may continue (``can_force``).
|
||||
|
||||
Step 2's tries follow the CLI's count (`phase.step2_attempts`). Step 4 is
|
||||
never graded: its only ``grade`` is a detected LeetCode Accepted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Final
|
||||
|
||||
from leetcode_coach import phase
|
||||
|
||||
NAMES: Final = {
|
||||
1: "constraint",
|
||||
2: "requirements",
|
||||
3: "toolbox",
|
||||
4: "implement",
|
||||
5: "postmortem",
|
||||
}
|
||||
FORCE_AFTER: Final = 3
|
||||
IMPLEMENT: Final = 4
|
||||
REQUIREMENTS: Final = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Step:
|
||||
"""One step as the state shows it."""
|
||||
|
||||
n: int
|
||||
status: str
|
||||
attempts: int
|
||||
last: dict[str, Any] | None
|
||||
|
||||
@property
|
||||
def done(self) -> bool:
|
||||
"""Passed or forced: the next step is open."""
|
||||
return self.status in {"passed", "forced"}
|
||||
|
||||
@property
|
||||
def can_force(self) -> bool:
|
||||
"""Open, graded, and at least `FORCE_AFTER` tries without a pass."""
|
||||
return (
|
||||
self.status == "open"
|
||||
and self.n != IMPLEMENT
|
||||
and self.attempts >= FORCE_AFTER
|
||||
)
|
||||
|
||||
|
||||
def _grades(events: list[dict[str, Any]], n: int) -> list[dict[str, Any]]:
|
||||
return [e for e in events if e.get("event") == "grade" and e.get("step") == n]
|
||||
|
||||
|
||||
def _last(grades: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
tried = [g for g in grades if g.get("forced") is not True]
|
||||
if not tried:
|
||||
return None
|
||||
g = tried[-1]
|
||||
return {
|
||||
"grade": g.get("grade"),
|
||||
"feedback": g.get("feedback", ""),
|
||||
"via": g.get("via"),
|
||||
"at": g.get("at"),
|
||||
}
|
||||
|
||||
|
||||
def _status(grades: list[dict[str, Any]], *, reachable: bool) -> str:
|
||||
if any(g.get("forced") is True for g in grades):
|
||||
return "forced"
|
||||
if any(g.get("grade") == "pass" for g in grades):
|
||||
return "passed"
|
||||
return "open" if reachable else "locked"
|
||||
|
||||
|
||||
def derive(events: list[dict[str, Any]]) -> list[Step]:
|
||||
"""Every step of the session whose events these are."""
|
||||
steps: list[Step] = []
|
||||
reachable = True
|
||||
for n in NAMES:
|
||||
grades = _grades(events, n)
|
||||
status = _status(grades, reachable=reachable)
|
||||
if n == REQUIREMENTS: # the CLI's count, so both clients agree
|
||||
attempts = phase.step2_attempts(events)
|
||||
else:
|
||||
attempts = sum(
|
||||
1
|
||||
for g in grades
|
||||
if g.get("grade") != "pass" and g.get("forced") is not True
|
||||
)
|
||||
step = Step(n, status, attempts, _last(grades))
|
||||
steps.append(step)
|
||||
reachable = step.done
|
||||
return steps
|
||||
|
||||
|
||||
def open_step(steps: list[Step]) -> Step | None:
|
||||
"""The step kuhy is on, or None when every step is done."""
|
||||
return next((s for s in steps if s.status == "open"), None)
|
||||
|
||||
|
||||
def rung(events: list[dict[str, Any]]) -> str | None:
|
||||
"""The coach's current hint rung (its last marker)."""
|
||||
rungs = [e.get("rung") for e in events if e.get("event") == "rung"]
|
||||
return str(rungs[-1]) if rungs else None
|
||||
|
||||
|
||||
def revealed(events: list[dict[str, Any]]) -> bool:
|
||||
"""Whether kuhy typed ``reveal`` in this session."""
|
||||
return any(e.get("event") == "reveal" for e in events)
|
||||
|
||||
|
||||
def view(step: Step, probe: str | None, probed_at: int | None) -> dict[str, Any]:
|
||||
"""The step's row in the published state (section 3)."""
|
||||
row: dict[str, Any] = {"n": step.n, "name": NAMES[step.n], "status": step.status}
|
||||
if step.n == IMPLEMENT:
|
||||
return {**row, "probe": probe, "probed_at": probed_at}
|
||||
return {
|
||||
**row,
|
||||
"attempts": step.attempts,
|
||||
"can_force": step.can_force,
|
||||
"last": step.last,
|
||||
}
|
||||
105
leetcode_coach/leetcode_coach/relay/toolbox_grade.py
Normal file
105
leetcode_coach/leetcode_coach/relay/toolbox_grade.py
Normal file
@ -0,0 +1,105 @@
|
||||
"""Step 3, the toolbox pick, graded mechanically against the answer key.
|
||||
|
||||
No model. Key entries `K` (``entries``), picks `P`. An entry in
|
||||
``also_accepted`` is a correct alternative: each one picked stands in for
|
||||
one key entry not picked (sum instead of XOR for Missing Number). A combined
|
||||
key entry ("prefix sum + hash map") also counts as picked when every one of
|
||||
its parts is: each part must resolve to another toolbox entry by name or
|
||||
alias, ignoring case and a plural ``s`` ("prefix sum" -> "prefix sums"); a
|
||||
part that names no entry ("merge") means only the combined entry counts.
|
||||
|
||||
* pass: every key entry is covered (picked, built from its parts, or stood
|
||||
in for) and at most one pick is neither key, part nor alternative;
|
||||
* partial: something right was picked, but not that;
|
||||
* fail: nothing right was picked.
|
||||
|
||||
Feedback is the properties the uncovered key entries need that no pick has,
|
||||
as property names (`phase.property_line`), never an entry name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from leetcode_coach import phase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from leetcode_coach.answer_key import Answer
|
||||
from leetcode_coach.toolbox import Entry, Toolbox
|
||||
|
||||
MAX_EXTRAS: Final = 1
|
||||
MIN_PARTS: Final = 2
|
||||
SPREAD: Final = (
|
||||
"Your picks have the properties, but spread over tools that do not "
|
||||
"combine; find the tool that has them together."
|
||||
)
|
||||
TOO_MANY: Final = "Fewer picks: more than one of these is not needed."
|
||||
|
||||
|
||||
def _key(text: str) -> str:
|
||||
return " ".join(text.split()).casefold()
|
||||
|
||||
|
||||
def resolve(name: str, box: Toolbox) -> Entry | None:
|
||||
"""The entry called `name` (or with that alias)."""
|
||||
wanted = _key(name)
|
||||
return next(
|
||||
(e for e in box.entries if wanted in {_key(n) for n in e.names()}), None
|
||||
)
|
||||
|
||||
|
||||
def parts(entry: Entry, box: Toolbox) -> tuple[Entry, ...] | None:
|
||||
"""The entries a combined ``a + b`` entry is made of; None if not all exist."""
|
||||
pieces = [p.strip() for p in entry.name.split(" + ")]
|
||||
if len(pieces) < MIN_PARTS: # a plain entry has no parts
|
||||
return None
|
||||
found: list[Entry] = []
|
||||
for piece in pieces:
|
||||
match = resolve(piece, box) or resolve(piece + "s", box)
|
||||
if match is None or match.name == entry.name:
|
||||
return None
|
||||
found.append(match)
|
||||
return tuple(found)
|
||||
|
||||
|
||||
def grade(picks: list[str], key: Answer, box: Toolbox) -> tuple[str, str]:
|
||||
"""``(grade, feedback)`` for `picks` (entry names, already resolvable)."""
|
||||
chosen = {e.name for p in picks if (e := resolve(p, box)) is not None}
|
||||
used: set[str] = set()
|
||||
missing: list[str] = []
|
||||
for name in key.entries:
|
||||
entry = resolve(name, box)
|
||||
made = None if entry is None else parts(entry, box)
|
||||
if name in chosen:
|
||||
used.add(name)
|
||||
elif made is not None and all(m.name in chosen for m in made):
|
||||
used.update(m.name for m in made)
|
||||
else:
|
||||
missing.append(name)
|
||||
stand_ins = chosen & set(key.also_accepted)
|
||||
extras = chosen - used - stand_ins
|
||||
if len(missing) <= len(stand_ins) and len(extras) <= MAX_EXTRAS:
|
||||
return "pass", ""
|
||||
verdict = "partial" if used or stand_ins else "fail"
|
||||
return verdict, _feedback(missing, chosen, extras, box)
|
||||
|
||||
|
||||
def _feedback(
|
||||
missing: list[str], chosen: set[str], extras: set[str], box: Toolbox
|
||||
) -> str:
|
||||
have = {p for e in box.entries if e.name in chosen for p in e.properties}
|
||||
need: list[str] = []
|
||||
for name in missing:
|
||||
entry = resolve(name, box)
|
||||
for prop in () if entry is None else entry.properties:
|
||||
if prop not in have and prop not in need:
|
||||
need.append(prop)
|
||||
lines = [f"- {phase.property_line(p, box)}" for p in need if p in box.properties]
|
||||
if lines:
|
||||
head = "The operation you need also has these properties, which no pick has:"
|
||||
text = "\n".join([head, *lines])
|
||||
else:
|
||||
text = SPREAD if missing else ""
|
||||
if len(extras) > MAX_EXTRAS:
|
||||
text = f"{text}\n{TOO_MANY}".strip()
|
||||
return text
|
||||
203
leetcode_coach/leetcode_coach/relay/turns.py
Normal file
203
leetcode_coach/leetcode_coach/relay/turns.py
Normal file
@ -0,0 +1,203 @@
|
||||
"""Session start, chat, hint, reveal and run-code (DOCS-coach-relay.md 2).
|
||||
|
||||
Every kuhy line is logged (``chat``) and shown before the model is asked,
|
||||
so the app sees it at once. Code -- typed in chat or sent to run -- is
|
||||
refused while the relay's own step 2 is neither passed nor forced, checked
|
||||
BEFORE `PhaseGate.screen`: the gate's own 3-incompletes auto-open must not
|
||||
skip kuhy's choice to force. Grades waiting for the coach ride as a prefix
|
||||
on its next turn; ``reveal`` goes out bare, because the Stop gate lifts only
|
||||
on a message that is exactly ``reveal``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import logging
|
||||
import secrets
|
||||
from typing import TYPE_CHECKING, Any, Final, NoReturn
|
||||
import urllib.error
|
||||
|
||||
from leetcode_coach import (
|
||||
fetch_problem,
|
||||
local_prompt,
|
||||
phase,
|
||||
progress,
|
||||
req_check,
|
||||
session_log,
|
||||
)
|
||||
from leetcode_coach.relay import protocol, steps
|
||||
from leetcode_coach.relay.errors import CoachError
|
||||
from leetcode_coach.relay.session import Session
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from leetcode_coach.relay.coach import Coach
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
|
||||
LOCKED: Final = "Code waits until step 2 (requirements) is passed or forced."
|
||||
RUN_ERRORS: Final = {
|
||||
"timeout": "Your code ran past its time limit.",
|
||||
"node_missing": "node is not installed on the PC.",
|
||||
"no_examples": "No examples could be read from the statement.",
|
||||
"syntax": "Your code does not parse (SyntaxError).",
|
||||
}
|
||||
|
||||
|
||||
def fetch_statement(slug: str) -> tuple[str, str] | None:
|
||||
"""``(title, statement)`` from LeetCode, in-process; None when unavailable."""
|
||||
try:
|
||||
question = fetch_problem.fetch(slug)
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
||||
_logger.warning("could not fetch %s: %s", slug, exc)
|
||||
return None
|
||||
if question is None or question.get("isPaidOnly") or not question.get("content"):
|
||||
_logger.warning("%s is unknown or premium-only", slug)
|
||||
return None
|
||||
return str(question["title"]), fetch_problem.render(question, "javascript")
|
||||
|
||||
|
||||
def gate_for(coach: Coach, log: Path, slug: str) -> phase.PhaseGate:
|
||||
"""The phase gate for one session's log, built directly (never ``open``)."""
|
||||
return phase.PhaseGate(log, coach.deps.box(), lambda: slug, coach.deps.ask)
|
||||
|
||||
|
||||
def statement(coach: Coach, session: Session) -> str | None:
|
||||
"""The session's statement, fetched again if a restart lost it."""
|
||||
if session.statement is None and (got := coach.deps.fetch(session.slug)):
|
||||
session.statement = got[1]
|
||||
session.gate.known = got[1]
|
||||
return session.statement
|
||||
|
||||
|
||||
def _sid(now_ms: int) -> str:
|
||||
local = datetime.fromtimestamp(now_ms / 1000, tz=UTC).astimezone()
|
||||
return f"{local:%Y%m%d-%H%M%S}-{secrets.token_hex(2)}"
|
||||
|
||||
|
||||
def start(coach: Coach, req: protocol.Request) -> dict[str, Any]:
|
||||
"""Open a session for ``body.slug``: the coach's opener first."""
|
||||
live = coach.session
|
||||
if live is not None and live.ended is None:
|
||||
msg = "A session is already live; end it first."
|
||||
raise CoachError(msg, code="session_live")
|
||||
slug = str(req.body["slug"])
|
||||
if slug not in coach.deps.answers():
|
||||
msg = f"{slug} has no answer key: only the cluster problems are coached here."
|
||||
raise CoachError(msg, code="no_answer_key")
|
||||
fetched = coach.deps.fetch(slug)
|
||||
now = coach.now()
|
||||
sid = _sid(now)
|
||||
day = datetime.fromtimestamp(now / 1000, tz=UTC).astimezone().date()
|
||||
log = session_log.log_path(slug, day)
|
||||
session = Session(
|
||||
sid=sid,
|
||||
slug=slug,
|
||||
title=fetched[0] if fetched else slug,
|
||||
url=f"https://leetcode.com/problems/{slug}/",
|
||||
cluster=progress.cluster_of(slug),
|
||||
log=log,
|
||||
started=now,
|
||||
transport=req.transport,
|
||||
dev=req.dev,
|
||||
gate=coach.gate_for(log, slug),
|
||||
statement=fetched[1] if fetched else None,
|
||||
)
|
||||
session.gate.known = session.statement
|
||||
raw = coach.chat.open(session)
|
||||
session.log_event(
|
||||
"session_start",
|
||||
source="app",
|
||||
transport=req.transport,
|
||||
dev=req.dev,
|
||||
slug=slug,
|
||||
at=now,
|
||||
)
|
||||
key, _rung = _reply(coach, session, raw, "chat")
|
||||
coach.session = session
|
||||
return {"session": sid, "msg": key}
|
||||
|
||||
|
||||
def _kuhy(coach: Coach, session: Session, kind: str, text: str) -> None:
|
||||
now = coach.now()
|
||||
session.log_event("chat", role="kuhy", kind=kind, text=text, at=now)
|
||||
session.add("kuhy", kind, text, now)
|
||||
|
||||
|
||||
def _reply(
|
||||
coach: Coach, session: Session, raw: str, kind: str
|
||||
) -> tuple[str, str | None]:
|
||||
"""Log the reply's rung, strip it, show and log the reply."""
|
||||
rung = local_prompt.split_rung(raw)[1]
|
||||
clean = session.gate.after_reply(raw)
|
||||
now = coach.now()
|
||||
session.log_event("chat", role="coach", kind=kind, text=clean, at=now)
|
||||
return session.add("coach", kind, clean, now), rung
|
||||
|
||||
|
||||
def _prefixed(session: Session, text: str) -> str:
|
||||
if not session.notes:
|
||||
return text
|
||||
notes = "\n".join(session.notes)
|
||||
session.notes.clear()
|
||||
return f"{notes}\n\n{text}"
|
||||
|
||||
|
||||
def code_open(session: Session) -> bool:
|
||||
"""Whether the relay's own step 2 is passed or forced."""
|
||||
return steps.derive(session.events())[1].done
|
||||
|
||||
|
||||
def _refuse(coach: Coach, session: Session, note: str) -> NoReturn:
|
||||
session.add("system", "refused", f"{LOCKED}\n{note}".strip(), coach.now())
|
||||
raise CoachError(LOCKED, code="phase_locked", note=note)
|
||||
|
||||
|
||||
def chat(coach: Coach, session: Session, text: str) -> dict[str, Any]:
|
||||
"""One chat turn; ``hint`` and ``reveal`` typed alone are those kinds."""
|
||||
word = text.strip()
|
||||
if word in {"hint", "reveal"}:
|
||||
_kuhy(coach, session, word, word)
|
||||
coach.publish()
|
||||
if word == "hint":
|
||||
key, rung = _reply(
|
||||
coach, session, coach.chat.turn(session, _prefixed(session, word)), word
|
||||
)
|
||||
return {"msg": key, "rung": rung}
|
||||
session.gate.screen(word) # logs ``reveal``, as the REPL does
|
||||
session.notes.clear()
|
||||
key, _rung = _reply(coach, session, coach.chat.turn(session, word), word)
|
||||
coach.end(session, "reveal")
|
||||
return {"msg": key}
|
||||
is_code = phase.regex_verdict(word)
|
||||
if is_code is None:
|
||||
is_code = req_check.looks_like_code(word, coach.deps.ask)
|
||||
_kuhy(coach, session, "chat", text)
|
||||
if is_code and not code_open(session):
|
||||
_refuse(coach, session, session.gate.refusal_note())
|
||||
sent, note = session.gate.screen(text, is_code=is_code)
|
||||
if sent is None:
|
||||
_refuse(coach, session, note)
|
||||
if note:
|
||||
session.add("system", "grade", note, coach.now())
|
||||
coach.publish()
|
||||
raw = coach.chat.turn(session, _prefixed(session, sent))
|
||||
key, _rung = _reply(coach, session, raw, "chat")
|
||||
return {"msg": key}
|
||||
|
||||
|
||||
def run_code(coach: Coach, session: Session, code: str) -> dict[str, Any]:
|
||||
"""Run kuhy's JavaScript against the statement's examples."""
|
||||
if not code_open(session):
|
||||
_refuse(coach, session, session.gate.refusal_note())
|
||||
session.log_event("code", via="run", at=coach.now())
|
||||
result = coach.deps.run_code(code, statement(coach, session) or "")
|
||||
lines = [
|
||||
f"example {r['n']}: got {r['got']}, expected {r['expected']}"
|
||||
for r in result.get("examples", [])
|
||||
]
|
||||
if result.get("error"):
|
||||
lines.append(RUN_ERRORS.get(str(result["error"]), str(result["error"])))
|
||||
session.add("system", "run", "\n".join(lines), coach.now())
|
||||
return result
|
||||
@ -34,6 +34,8 @@ if TYPE_CHECKING:
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
HAIKU_TIMEOUT_S = 50.0
|
||||
LOCAL_TIMEOUT_S = 200.0
|
||||
"""Ollama's share of a check: with haiku's 50 s it fits the relay's 280 s cap."""
|
||||
HAIKU_ARGS = [
|
||||
"--model",
|
||||
"haiku",
|
||||
@ -113,7 +115,7 @@ def check_prompt(statement: str, req: str, box: Toolbox) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _ask_models[T](
|
||||
def ask_models[T](
|
||||
system: str,
|
||||
prompt: str,
|
||||
validate: Callable[[dict[str, Any] | None], T | None],
|
||||
@ -130,7 +132,11 @@ def _ask_models[T](
|
||||
return got, "haiku"
|
||||
try:
|
||||
local = local_backend.chat(
|
||||
[{"role": "system", "content": system}, {"role": "user", "content": prompt}]
|
||||
[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
timeout=LOCAL_TIMEOUT_S,
|
||||
)
|
||||
except (urllib.error.URLError, OSError, local_backend.ContextOverflowError) as exc:
|
||||
_LOG.warning("Ollama could not answer the check either: %s", exc)
|
||||
@ -149,7 +155,7 @@ def check(
|
||||
if not box.properties:
|
||||
_LOG.warning("no property vocabulary: the req list stays unverified")
|
||||
return Check("unverified")
|
||||
got = _ask_models(
|
||||
got = ask_models(
|
||||
CHECK_SYSTEM,
|
||||
check_prompt(statement, req, box),
|
||||
lambda obj: parse_verdict(obj, box.properties),
|
||||
@ -170,5 +176,5 @@ def looks_like_code(
|
||||
block: str, ask: Callable[..., dict[str, Any]] | None = None
|
||||
) -> bool:
|
||||
"""The model's call on a block the regex could not decide; prose if unsure."""
|
||||
got = _ask_models(CODE_SYSTEM, block, _code_flag, ask)
|
||||
got = ask_models(CODE_SYSTEM, block, _code_flag, ask)
|
||||
return False if got is None else got[0]
|
||||
|
||||
@ -6,7 +6,11 @@ directory follows ``LEETCODE_COACH_SESSIONS_DIR``: `progress.sessions_dir`).
|
||||
Every line is one object with ``event`` and ``ts``; ``req_check`` carries ``verdict``,
|
||||
``rung`` carries ``rung``. That contract is FIXED: ``leetcode_guard._coach_log``
|
||||
reads it for the status app, so a new kind of fact rides as an extra field on
|
||||
an existing event (``code`` + ``req_incomplete``), never as a new event name.
|
||||
an existing event (``code`` + ``req_incomplete``), and new names only together
|
||||
with the reader: ``session_start``, ``grade``, ``chat`` and ``session_end``
|
||||
came with ``_coach_log.EVENTS`` in one commit (DOCS-coach-relay.md section 5).
|
||||
A file can hold several app sessions; each starts at its ``session_start``
|
||||
(`current`), and a CLI-only file is one session.
|
||||
|
||||
Every event carries ``cluster`` while ``LEETCODE_COACH_CLUSTER`` is set
|
||||
(`leetcode --cluster` sets it for the session it starts).
|
||||
@ -32,7 +36,10 @@ _LOG = logging.getLogger(__name__)
|
||||
LOG_ENV = "LEETCODE_COACH_LOG"
|
||||
"""Set by the REPL so the Stop hook (a grandchild) reads the same log."""
|
||||
EVENTS = frozenset(
|
||||
{"req", "req_check", "code", "rung", "reveal", "postmortem", "retest"}
|
||||
{
|
||||
*("req", "req_check", "code", "rung", "reveal", "postmortem", "retest"),
|
||||
*("session_start", "grade", "chat", "session_end"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@ -95,9 +102,15 @@ def read(path: Path) -> list[dict[str, Any]]:
|
||||
return [obj for obj in parsed if isinstance(obj, dict)]
|
||||
|
||||
|
||||
def current(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""The events of the file's latest session: from its last ``session_start``."""
|
||||
starts = [i for i, e in enumerate(events) if e.get("event") == "session_start"]
|
||||
return events[starts[-1] :] if starts else events
|
||||
|
||||
|
||||
def has_code(path: Path) -> bool:
|
||||
"""Whether kuhy's code has reached the coach in this session."""
|
||||
return any(e.get("event") == "code" for e in read(path))
|
||||
return any(e.get("event") == "code" for e in current(read(path)))
|
||||
|
||||
|
||||
def append_current(event: str, **fields: object) -> bool:
|
||||
@ -107,8 +120,8 @@ def append_current(event: str, **fields: object) -> bool:
|
||||
logging ``postmortem`` once the cards are delivered. The TUI has no REPL
|
||||
and so no log: nothing is written there.
|
||||
"""
|
||||
current = os.environ.get(LOG_ENV)
|
||||
if not current:
|
||||
exported = os.environ.get(LOG_ENV)
|
||||
if not exported:
|
||||
return False
|
||||
append(Path(current), event, **fields)
|
||||
append(Path(exported), event, **fields)
|
||||
return True
|
||||
|
||||
@ -6,7 +6,8 @@ A call made with ``--model haiku`` is the checker; any other is the coach.
|
||||
Each call takes the next answer of its kind -- ``{"result": "..."}`` is printed
|
||||
the way ``--output-format json`` prints it, ``{"exit": n, "stderr": "..."}``
|
||||
fails -- and appends ``{"kind", "args", "prompt"}`` to ``calls.jsonl`` so a
|
||||
test can prove what did and did not reach the model.
|
||||
test can prove what did and did not reach the model, and with which
|
||||
``LEETCODE_COACH_LOG`` in its env.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@ -25,6 +26,7 @@ def main() -> int:
|
||||
kind = "haiku" if model == "haiku" else "coach"
|
||||
with (root / "calls.jsonl").open("a", encoding="utf-8") as fh:
|
||||
record = {"kind": kind, "args": args[:-1], "prompt": args[-1]}
|
||||
record["log"] = os.environ.get("LEETCODE_COACH_LOG", "")
|
||||
fh.write(json.dumps(record) + "\n")
|
||||
counter = root / f"{kind}.used"
|
||||
used = int(counter.read_text(encoding="utf-8")) if counter.exists() else 0
|
||||
|
||||
206
leetcode_coach/leetcode_coach/tests/_fake_rtdb.py
Normal file
206
leetcode_coach/leetcode_coach/tests/_fake_rtdb.py
Normal file
@ -0,0 +1,206 @@
|
||||
"""An in-process fake Firebase RTDB: REST (GET/PUT/PATCH/DELETE) plus SSE streams.
|
||||
|
||||
Enough of RTDB's semantics for the relay's tests: a JSON tree where a ``null``
|
||||
write deletes and an empty object vanishes, ``{".sv": "timestamp"}`` resolved to
|
||||
`now_ms`, ``print=silent`` answered 204, ``?auth=`` required. A GET with
|
||||
``Accept: text/event-stream`` opens a stream that starts with a ``put`` of the
|
||||
whole node at ``/`` and then reports every later write at or under it -- over
|
||||
HTTP/1.1 chunked encoding, as Firebase sends it, so a client that buffers until
|
||||
EOF would hang here too. `write` is the phone's side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from http.server import ThreadingHTTPServer
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any, Self
|
||||
|
||||
from leetcode_coach.tests._fake_rtdb_handler import handler_for, segments
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from types import TracebackType
|
||||
|
||||
ROOT = "leetcode-guard/coach"
|
||||
NOW_MS = 1_792_771_488_000
|
||||
Item = tuple[str, str] | None
|
||||
|
||||
|
||||
def _clean(value: Any, now: int) -> Any:
|
||||
"""Resolve server timestamps and drop nulls and empty objects."""
|
||||
if isinstance(value, dict):
|
||||
if value == {".sv": "timestamp"}:
|
||||
return now
|
||||
kept = {k: c for k, v in value.items() if (c := _clean(v, now)) is not None}
|
||||
return kept or None
|
||||
return value
|
||||
|
||||
|
||||
def _set_in(node: dict[str, Any], segs: list[str], value: Any) -> dict[str, Any]:
|
||||
"""A copy of `node` with `value` at `segs`; emptied objects pruned."""
|
||||
head, rest = segs[0], segs[1:]
|
||||
if rest:
|
||||
child = node.get(head)
|
||||
here = _set_in(child if isinstance(child, dict) else {}, rest, value) or None
|
||||
else:
|
||||
here = value
|
||||
out = dict(node)
|
||||
if here is None:
|
||||
out.pop(head, None)
|
||||
else:
|
||||
out[head] = here
|
||||
return out
|
||||
|
||||
|
||||
class _Stream:
|
||||
def __init__(self, path: list[str]) -> None:
|
||||
self.path = path
|
||||
self.items: queue.Queue[Item] = queue.Queue()
|
||||
|
||||
|
||||
class FakeRtdb:
|
||||
"""The fake database and its HTTP server."""
|
||||
|
||||
def __init__(self, now_ms: Callable[[], int] | None = None) -> None:
|
||||
self.now_ms = now_ms or (lambda: NOW_MS)
|
||||
self.tree: dict[str, Any] = {}
|
||||
self.tokens: list[str] = []
|
||||
self.requests: list[tuple[str, str, str]] = []
|
||||
self._failures: list[int] = []
|
||||
self._streams: list[_Stream] = []
|
||||
self._lock = threading.RLock()
|
||||
self._stopping = threading.Event()
|
||||
self._server = ThreadingHTTPServer(("127.0.0.1", 0), handler_for(self))
|
||||
self._server.daemon_threads = True
|
||||
self._thread = threading.Thread(
|
||||
target=self._server.serve_forever, args=(0.01,), daemon=True
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def start(cls, now_ms: Callable[[], int] | None = None) -> FakeRtdb:
|
||||
fake = cls(now_ms)
|
||||
fake._thread.start()
|
||||
return fake
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
host, port = self._server.server_address[:2]
|
||||
return f"http://{host!s}:{port}"
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
kind: type[BaseException] | None,
|
||||
exc: BaseException | None,
|
||||
tb: TracebackType | None,
|
||||
) -> None:
|
||||
self.stop()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stopping.set()
|
||||
with self._lock:
|
||||
for stream in self._streams:
|
||||
stream.items.put(None)
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
|
||||
# -- the tree ------------------------------------------------------------
|
||||
|
||||
def _get(self, segs: list[str]) -> Any:
|
||||
node: Any = self.tree
|
||||
for seg in segs:
|
||||
if not isinstance(node, dict) or seg not in node:
|
||||
return None
|
||||
node = node[seg]
|
||||
return node
|
||||
|
||||
def _set(self, segs: list[str], value: Any) -> None:
|
||||
if not segs:
|
||||
self.tree = value if isinstance(value, dict) else {}
|
||||
return
|
||||
self.tree = _set_in(self.tree, segs, value)
|
||||
|
||||
def node(self, segs: list[str]) -> Any:
|
||||
"""The value at absolute `segs` (``None`` when absent)."""
|
||||
with self._lock:
|
||||
return self._get(segs)
|
||||
|
||||
def read(self, path: str = "") -> Any:
|
||||
with self._lock:
|
||||
return json.loads(json.dumps(self._get(segments(f"{ROOT}/{path}"))))
|
||||
|
||||
def write(self, path: str, value: Any) -> None:
|
||||
"""The phone's PUT under the coach root."""
|
||||
self.apply("PUT", segments(f"{ROOT}/{path}"), value)
|
||||
|
||||
def apply(self, method: str, segs: list[str], value: Any) -> Any:
|
||||
now = self.now_ms()
|
||||
with self._lock:
|
||||
if method == "PATCH":
|
||||
if not isinstance(value, dict):
|
||||
raise TypeError
|
||||
patch = {k: _clean(v, now) for k, v in value.items()}
|
||||
for key, child in patch.items():
|
||||
self._set(segs + segments(key), child)
|
||||
self._notify(segs, "patch", patch)
|
||||
return patch
|
||||
stored = None if method == "DELETE" else _clean(value, now)
|
||||
self._set(segs, stored)
|
||||
self._notify(segs, "put", stored)
|
||||
return stored
|
||||
|
||||
def _notify(self, segs: list[str], kind: str, data: Any) -> None:
|
||||
for stream in self._streams:
|
||||
depth = len(stream.path)
|
||||
if segs[:depth] == stream.path:
|
||||
rel = "/" + "/".join(segs[depth:])
|
||||
stream.items.put((kind, json.dumps({"path": rel, "data": data})))
|
||||
elif stream.path[: len(segs)] == segs:
|
||||
whole = {"path": "/", "data": self._get(stream.path)}
|
||||
stream.items.put(("put", json.dumps(whole)))
|
||||
|
||||
# -- test controls -------------------------------------------------------
|
||||
|
||||
def fail_next(self, n: int, status: int = 500) -> None:
|
||||
with self._lock:
|
||||
self._failures += [status] * n
|
||||
|
||||
def take_failure(self) -> int | None:
|
||||
with self._lock:
|
||||
return self._failures.pop(0) if self._failures else None
|
||||
|
||||
def _end_streams(self, event: str, data: str) -> None:
|
||||
with self._lock:
|
||||
for stream in self._streams:
|
||||
stream.items.put((event, data))
|
||||
stream.items.put(None)
|
||||
|
||||
def revoke_streams(self) -> None:
|
||||
self._end_streams("auth_revoked", '"credential is no longer valid"')
|
||||
|
||||
def cancel_streams(self) -> None:
|
||||
self._end_streams("cancel", "null")
|
||||
|
||||
def stream_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._streams)
|
||||
|
||||
def open_stream(self, segs: list[str]) -> _Stream:
|
||||
stream = _Stream(segs)
|
||||
with self._lock:
|
||||
whole = {"path": "/", "data": self._get(segs)}
|
||||
stream.items.put(("put", json.dumps(whole)))
|
||||
self._streams.append(stream)
|
||||
return stream
|
||||
|
||||
def close_stream(self, stream: _Stream) -> None:
|
||||
with self._lock:
|
||||
self._streams.remove(stream)
|
||||
|
||||
@property
|
||||
def stopping(self) -> bool:
|
||||
return self._stopping.is_set()
|
||||
100
leetcode_coach/leetcode_coach/tests/_fake_rtdb_handler.py
Normal file
100
leetcode_coach/leetcode_coach/tests/_fake_rtdb_handler.py
Normal file
@ -0,0 +1,100 @@
|
||||
"""The HTTP side of `_fake_rtdb.FakeRtdb`: REST answers and chunked SSE streams."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from leetcode_coach.tests._fake_rtdb import FakeRtdb
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def segments(path: str) -> list[str]:
|
||||
"""``a/b%2Fc/`` -> ``["a", "b/c"]``."""
|
||||
return [unquote(p) for p in path.strip("/").split("/") if p]
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
"""REST and SSE answers from `fake` (set by `handler_for`)."""
|
||||
|
||||
fake: ClassVar[FakeRtdb]
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def log_message(self, fmt: str, /, *args: object) -> None:
|
||||
del fmt, args
|
||||
|
||||
def _reply(self, status: int, body: object = None) -> None:
|
||||
raw = b"" if status == 204 else json.dumps(body).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def _handle(self, method: str) -> None:
|
||||
split = urlsplit(self.path)
|
||||
query = parse_qs(split.query)
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
raw = self.rfile.read(length) if length else b""
|
||||
self.fake.requests.append((method, split.path, split.query))
|
||||
token = query.get("auth", [""])[0]
|
||||
if not token:
|
||||
self._reply(401, {"error": "Permission denied"})
|
||||
return
|
||||
self.fake.tokens.append(token)
|
||||
if (status := self.fake.take_failure()) is not None:
|
||||
self._reply(status, {"error": "injected"})
|
||||
return
|
||||
segs = segments(split.path.removesuffix(".json"))
|
||||
if method == "GET" and "event-stream" in self.headers.get("Accept", ""):
|
||||
self._stream(segs)
|
||||
return
|
||||
if method == "GET":
|
||||
self._reply(200, self.fake.node(segs))
|
||||
return
|
||||
stored = self.fake.apply(method, segs, json.loads(raw) if raw else None)
|
||||
silent = query.get("print", [""])[0] == "silent"
|
||||
self._reply(204 if silent or method == "DELETE" else 200, stored)
|
||||
|
||||
def _chunk(self, text: str) -> None:
|
||||
data = text.encode()
|
||||
self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n")
|
||||
self.wfile.flush()
|
||||
|
||||
def _stream(self, segs: list[str]) -> None:
|
||||
self.close_connection = True
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.send_header("Transfer-Encoding", "chunked")
|
||||
self.end_headers()
|
||||
stream = self.fake.open_stream(segs)
|
||||
try:
|
||||
while (item := stream.items.get()) is not None:
|
||||
self._chunk(f"event: {item[0]}\ndata: {item[1]}\n\n")
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
except OSError as exc:
|
||||
_LOG.warning("fake RTDB stream: the client hung up (%s)", exc)
|
||||
finally:
|
||||
self.fake.close_stream(stream)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._handle("GET")
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
self._handle("PUT")
|
||||
|
||||
def do_PATCH(self) -> None:
|
||||
self._handle("PATCH")
|
||||
|
||||
def do_DELETE(self) -> None:
|
||||
self._handle("DELETE")
|
||||
|
||||
|
||||
def handler_for(fake: FakeRtdb) -> type[BaseHTTPRequestHandler]:
|
||||
"""A request handler class bound to `fake`."""
|
||||
return type("Handler", (_Handler,), {"fake": fake})
|
||||
64
leetcode_coach/leetcode_coach/tests/_relay_direct.py
Normal file
64
leetcode_coach/leetcode_coach/tests/_relay_direct.py
Normal file
@ -0,0 +1,64 @@
|
||||
"""The backend without threads or HTTP: a scripted in-process ``ask``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from leetcode_coach.relay import protocol
|
||||
from leetcode_coach.relay.coach import Coach, Deps
|
||||
from leetcode_coach.relay.probe import AcceptedProbe
|
||||
from leetcode_coach.tests._relay_statements import MISSING_NUMBER
|
||||
from leetcode_coach.tests._relay_support import DEV, Clock, FakeProbe, rid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class FakeAsk:
|
||||
"""``claude_cli.ask`` in-process: scripted answers by kind, every call kept."""
|
||||
|
||||
def __init__(
|
||||
self, coach: list[dict[str, Any]], haiku: list[dict[str, Any]] | None = None
|
||||
) -> None:
|
||||
self.script = {"coach": list(coach), "haiku": list(haiku or [])}
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
def __call__(self, args: list[str], prompt: str, **kw: Any) -> dict[str, Any]:
|
||||
kind = "haiku" if "haiku" in args else "coach"
|
||||
self.calls.append({"kind": kind, "args": args, "prompt": prompt, **kw})
|
||||
answer = self.script[kind].pop(0) if self.script[kind] else {"result": "?"}
|
||||
return {"session_id": "s1", **answer} if kind == "coach" else answer
|
||||
|
||||
def prompts(self, kind: str = "coach") -> list[str]:
|
||||
"""What reached the model of `kind`."""
|
||||
return [c["prompt"] for c in self.calls if c["kind"] == kind]
|
||||
|
||||
|
||||
def make_coach(tmp_path: Path, ask: FakeAsk, **deps: Any) -> Coach:
|
||||
"""A backend with no threads, a fake ask and dry-run cards."""
|
||||
base: dict[str, Any] = {
|
||||
"state_dir": tmp_path / "relay",
|
||||
"ask": ask,
|
||||
"fetch": lambda _slug: ("Missing Number", MISSING_NUMBER),
|
||||
"probe": AcceptedProbe(FakeProbe(), monotonic=Clock()),
|
||||
"cards_dry_run": True,
|
||||
}
|
||||
return Coach(Deps(**(base | deps)))
|
||||
|
||||
|
||||
def req(kind: str, transport: str = "desktop", **body: object) -> protocol.Request:
|
||||
"""A validated request, as the lane hands it over."""
|
||||
raw = {"rid": rid(), "kind": kind, "dev": DEV, "body": body}
|
||||
return protocol.parse(raw, transport)
|
||||
|
||||
|
||||
def started(coach: Coach) -> str:
|
||||
"""Start Missing Number (the opener must already be scripted); its sid."""
|
||||
result = coach.handle(req("session_start", slug="missing-number"))
|
||||
assert result["ok"], result
|
||||
return str(result["data"]["session"])
|
||||
|
||||
|
||||
def grade(coach: Coach, sid: str, step: int, **body: Any) -> dict[str, Any]:
|
||||
"""One ``grade_step`` result."""
|
||||
return coach.handle(req("grade_step", session=sid, step=step, **body))
|
||||
185
leetcode_coach/leetcode_coach/tests/_relay_flow.py
Normal file
185
leetcode_coach/leetcode_coach/tests/_relay_flow.py
Normal file
@ -0,0 +1,185 @@
|
||||
"""The spec's scripted session, run the same way over either transport.
|
||||
|
||||
TODO-coach-ui.md "Done": start Missing Number, each graded step answers
|
||||
pass/partial/fail with feedback, the next step is blocked until a pass or
|
||||
the 3rd try, the chat round-trips, Hint logs a rung, run-code runs on real
|
||||
node, a simulated Accepted passes step 4 within one poll, the post-mortem is
|
||||
graded and its cards built (dry-run), and the Coach-tab history shows the
|
||||
four grades and the highest rung. `Ends` is how one transport sends a
|
||||
request and reads ``state`` / ``chat``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from leetcode_coach import answer_key, session_log
|
||||
from leetcode_coach.tests import _relay_support as rs
|
||||
from leetcode_guard import _coach_log
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
OPENER = "Let's start. What limit binds here?\n<!-- rung:H0 -->"
|
||||
HINT = "What happens to a value combined with itself?\n<!-- rung:H1 -->"
|
||||
CODE = (
|
||||
"var missingNumber = function(nums) {\n let x = nums.length;\n"
|
||||
" for (let i = 0; i < nums.length; i++) x ^= i ^ nums[i];\n return x;\n};"
|
||||
)
|
||||
|
||||
|
||||
def _haiku(**obj: object) -> dict[str, str]:
|
||||
return {"result": json.dumps(obj)}
|
||||
|
||||
|
||||
SCRIPT: dict[str, list[dict[str, Any]]] = {
|
||||
"haiku": [
|
||||
_haiku(grade="fail", feedback="Say what the space limit rules out."),
|
||||
_haiku(grade="partial", feedback="Now say what that forces."),
|
||||
_haiku(grade="fail", feedback="Name the limit first."),
|
||||
_haiku(verdict="incomplete", missing=["self-inverse"]),
|
||||
_haiku(verdict="pass", missing=[]),
|
||||
_haiku(grade="pass", feedback=""),
|
||||
],
|
||||
"coach": [
|
||||
{"result": OPENER},
|
||||
{"result": "Which limit did you pick, and why?"},
|
||||
{"result": HINT},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Ends:
|
||||
"""One transport: send a request, read the state, read chat lines."""
|
||||
|
||||
send: Callable[..., dict[str, Any]]
|
||||
state: Callable[[], dict[str, Any]]
|
||||
chat: Callable[[str | None], dict[str, Any]]
|
||||
|
||||
def step(self, n: int) -> dict[str, Any]:
|
||||
"""Step `n`'s row in the current state."""
|
||||
return dict(self.state()["steps"][n - 1])
|
||||
|
||||
|
||||
def _through_step_two(rig: rs.Rig, ends: Ends, sid: str) -> None:
|
||||
send = ends.send
|
||||
assert send("session_start", slug="two-sum")["code"] == "session_live"
|
||||
assert send("grade_step", session=sid, step=2, answer="x")["code"] == "step_locked"
|
||||
tries = [
|
||||
send("grade_step", session=sid, step=1, answer="n <= 10^4") for _ in range(3)
|
||||
]
|
||||
assert [t["data"]["grade"] for t in tries] == ["fail", "partial", "fail"]
|
||||
assert [t["data"]["attempt"] for t in tries] == [1, 2, 3]
|
||||
assert [t["data"]["can_force"] for t in tries] == [False, False, True]
|
||||
assert tries[0]["data"]["feedback"] == "Say what the space limit rules out."
|
||||
forced = send("grade_step", session=sid, step=1, force=True)
|
||||
assert forced["data"] == {"step": 1, "forced": True, "next_step": 2}
|
||||
|
||||
reply = send("chat", session=sid, text="Is it the memory limit?")
|
||||
prompts = [
|
||||
c["prompt"] for c in rs.calls(rig.extra["claude"]) if c["kind"] == "coach"
|
||||
]
|
||||
assert prompts[1].startswith("[app: step 1 graded fail on try 1; step 1 open]")
|
||||
assert "[app: step 1 continued without a pass on try 3; step 2 open]" in prompts[1]
|
||||
assert prompts[1].endswith("Is it the memory limit?")
|
||||
lines = ends.chat("m0001")
|
||||
assert lines[reply["data"]["msg"]]["text"] == "Which limit did you pick, and why?"
|
||||
assert any(v["kind"] == "grade" and v["role"] == "system" for v in lines.values())
|
||||
|
||||
early = send("chat", session=sid, text="const seen = new Set();")
|
||||
assert early["code"] == "phase_locked"
|
||||
assert "requirement list" in early["note"]
|
||||
assert send("run_code", session=sid, code=CODE)["code"] == "phase_locked"
|
||||
partial = send("grade_step", session=sid, step=2, answer="constant space, one pass")
|
||||
assert partial["data"]["grade"] == "partial"
|
||||
assert "self-inverse" in partial["data"]["feedback"]
|
||||
passed = send("grade_step", session=sid, step=2, answer="self-inverse, commutative")
|
||||
assert passed["data"] == {
|
||||
"step": 2,
|
||||
"grade": "pass",
|
||||
"feedback": "",
|
||||
"attempt": 2,
|
||||
"can_force": False,
|
||||
"via": "haiku",
|
||||
"next_step": 3,
|
||||
}
|
||||
|
||||
|
||||
def _toolbox_to_accepted(rig: rs.Rig, ends: Ends, sid: str) -> None:
|
||||
send = ends.send
|
||||
state = ends.state()
|
||||
assert state["step"] == 3
|
||||
assert {e["name"] for e in state["toolbox"]} == {"XOR", "hash map", "sum formula"}
|
||||
assert "XOR" not in json.dumps(state["steps"])
|
||||
wrong = send("grade_step", session=sid, step=3, picks=["hash map"])
|
||||
assert wrong["data"]["grade"] == "fail"
|
||||
assert "XOR" not in wrong["data"]["feedback"]
|
||||
assert "self-inverse" in wrong["data"]["feedback"]
|
||||
assert (
|
||||
send("grade_step", session=sid, step=3, picks=["XOR"])["data"]["next_step"] == 4
|
||||
)
|
||||
|
||||
hint = send("hint", session=sid)
|
||||
assert hint["data"]["rung"] == "H1"
|
||||
assert ends.state()["rung"] == "H1"
|
||||
ran = send("run_code", session=sid, code=CODE)
|
||||
assert ran["data"]["error"] is None
|
||||
assert [e["ok"] for e in ran["data"]["examples"]] == [True, True, True]
|
||||
|
||||
rs.wait_for(lambda: ends.step(4)["probe"] == "not_yet")
|
||||
assert send("grade_step", session=sid, step=5, answer="x")["code"] == "step_locked"
|
||||
assert send("grade_step", session=sid, step=4, force=True)["code"] == "cannot_force"
|
||||
session = rig.relay.coach.session
|
||||
assert session is not None
|
||||
rig.probe.status = "accepted"
|
||||
rig.probe.timestamp_s = session.started // 1000 + 5
|
||||
rig.clock.now += 60
|
||||
rs.wait_for(lambda: ends.step(4)["status"] == "passed", timeout=2)
|
||||
assert ends.step(4)["probe"] == "accepted"
|
||||
assert ends.step(5)["status"] == "open"
|
||||
|
||||
|
||||
def run_flow(rig: rs.Rig, ends: Ends) -> None:
|
||||
"""The whole session; every assertion of the spec's scripted E2E."""
|
||||
before = rs.guard_digest()
|
||||
start = ends.send("session_start", slug="missing-number")
|
||||
assert start["ok"], start
|
||||
sid = start["data"]["session"]
|
||||
assert start["data"]["msg"] == "m0001"
|
||||
first = rs.calls(rig.extra["claude"])[0]
|
||||
assert first["prompt"] == "/leetcode missing-number"
|
||||
log = session_log.sessions_dir() / first["log"].rsplit("/", 1)[1]
|
||||
assert first["log"] == str(log)
|
||||
_through_step_two(rig, ends, sid)
|
||||
_toolbox_to_accepted(rig, ends, sid)
|
||||
post = ends.send("grade_step", session=sid, step=5, answer="pairs cancel out")
|
||||
assert post["data"]["grade"] == "pass"
|
||||
assert post["data"]["cards"] == {"built": 1, "deck": "LeetCode", "error": None}
|
||||
assert post["data"]["next_step"] is None
|
||||
assert ends.state()["session"]["end_reason"] == "done"
|
||||
assert ends.send("chat", session=sid, text="thanks")["code"] == "not_live"
|
||||
|
||||
events = session_log.read(log)
|
||||
assert [e["event"] for e in events].count("grade") == 10
|
||||
assert all(e.get("sid") == sid for e in events if e["event"] in {"grade", "chat"})
|
||||
problem = _coach_log.read_session(log)
|
||||
assert problem is not None
|
||||
assert problem.max_rung == "H1"
|
||||
assert problem.accepted is True
|
||||
assert problem.grades is not None
|
||||
assert {k: v["grade"] for k, v in problem.grades.items()} == {
|
||||
"constraint": "fail",
|
||||
"requirements": "pass",
|
||||
"toolbox": "pass",
|
||||
"postmortem": "pass",
|
||||
}
|
||||
assert problem.grades["constraint"]["forced"] is True
|
||||
coach_calls = [c for c in rs.calls(rig.extra["claude"]) if c["kind"] == "coach"]
|
||||
assert all(c["log"] == str(log) for c in coach_calls)
|
||||
key = answer_key.load_answers()["missing-number"]
|
||||
assert key.design_rule not in log.read_text(encoding="utf-8")
|
||||
assert key.design_rule not in json.dumps(ends.chat(None))
|
||||
assert rs.guard_digest() == before
|
||||
86
leetcode_coach/leetcode_coach/tests/_relay_statements.py
Normal file
86
leetcode_coach/leetcode_coach/tests/_relay_statements.py
Normal file
@ -0,0 +1,86 @@
|
||||
"""Trimmed real statements (``fetch_problem.render``, fetched 2026-10-23).
|
||||
|
||||
Three shapes LeetCode uses for the examples: bold labels on plain lines
|
||||
(missing-number), bold labels with the values in code spans (two-sum), and
|
||||
bold labels inside a ``` fence (merge-intervals). The non-breaking-space
|
||||
lines are copied as fetched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
MISSING_NUMBER = """# 268. Missing Number (Easy)
|
||||
Tags: Array, Hash Table, Math, Binary Search, Bit Manipulation, Sorting
|
||||
|
||||
Given an array `nums` containing `n` distinct numbers in the range `[0, n]`.
|
||||
|
||||
**Example 1:**
|
||||
|
||||
**Input:** nums = [3,0,1]
|
||||
|
||||
**Output:** 2
|
||||
|
||||
**Explanation:**
|
||||
|
||||
`n = 3` since there are 3 numbers, so all numbers are in the range `[0,3]`.
|
||||
|
||||
**Example 2:**
|
||||
|
||||
**Input:** nums = [0,1]
|
||||
|
||||
**Output:** 2
|
||||
|
||||
**Example 3:**
|
||||
|
||||
**Input:** nums = [9,6,4,2,3,5,7,0,1]
|
||||
|
||||
**Output:** 8
|
||||
|
||||
## Starter code (javascript)
|
||||
```
|
||||
/**
|
||||
* @param {number[]} nums
|
||||
* @return {number}
|
||||
*/
|
||||
var missingNumber = function(nums) {
|
||||
|
||||
};
|
||||
```
|
||||
"""
|
||||
|
||||
TWO_SUM = """# 1. Two Sum (Easy)
|
||||
**Example 1:**
|
||||
**Input:** `nums = [2,7,11,15], target = 9`
|
||||
**Output:** `[0,1]`
|
||||
**Explanation:**
|
||||
The sum of `nums[0]` and `nums[1]` is `2 + 7 = 9`.
|
||||
**Example 2:**
|
||||
**Input:** `nums = [3,2,4], target = 6`
|
||||
**Output:** `[1,2]`
|
||||
**Example 3:**
|
||||
**Input:** `nums = [3,3], target = 6`
|
||||
**Output:** `[0,1]`
|
||||
## Starter code (javascript)
|
||||
```
|
||||
var twoSum = function(nums, target) {
|
||||
};
|
||||
```
|
||||
"""
|
||||
|
||||
MERGE_INTERVALS = """# 56. Merge Intervals (Medium)
|
||||
**Example 1:**
|
||||
```
|
||||
**Input:** intervals = [[1,3],[2,6],[8,10],[15,18]]
|
||||
**Output:** [[1,6],[8,10],[15,18]]
|
||||
**Explanation:** Since intervals [1,3] and [2,6] overlap, merge them into [1,6].
|
||||
```
|
||||
**Example 2:**
|
||||
```
|
||||
**Input:** intervals = [[1,4],[4,5]]
|
||||
**Output:** [[1,5]]
|
||||
```
|
||||
## Starter code (javascript)
|
||||
```
|
||||
var merge = function(intervals) {
|
||||
};
|
||||
```
|
||||
"""
|
||||
190
leetcode_coach/leetcode_coach/tests/_relay_support.py
Normal file
190
leetcode_coach/leetcode_coach/tests/_relay_support.py
Normal file
@ -0,0 +1,190 @@
|
||||
"""Shared plumbing for the relay tests: a fake claude, a fake probe, a client.
|
||||
|
||||
The relay runs in-process with its real threads (coverage sees it), on port
|
||||
0, with the 3 s pace and the minute between probes injected away. The
|
||||
LeetCode guard's data dir is redirected into the test's tmp dir and seeded
|
||||
(`guard_digest`), so "byte-identical afterwards" is checked without racing
|
||||
the live harvester on the developer's real files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import hashlib
|
||||
from http.client import HTTPConnection
|
||||
import itertools
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from leetcode_coach.relay.app import Relay
|
||||
from leetcode_coach.relay.coach import Deps
|
||||
from leetcode_coach.relay.probe import AcceptedProbe
|
||||
from leetcode_coach.tests._relay_statements import MISSING_NUMBER
|
||||
from leetcode_guard import _constants
|
||||
from leetcode_guard._submissions import AcSubmission, ProbeStatus, SolveProbe
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pytest
|
||||
|
||||
FAKE_CLAUDE = Path(__file__).with_name("_fake_claude.py")
|
||||
DEV = "3f2a7c1e-9b4d-4e6f-8a1b-2c3d4e5f6a7b"
|
||||
_RIDS = itertools.count(1)
|
||||
|
||||
|
||||
def rid() -> str:
|
||||
"""A fresh ULID-shaped request id."""
|
||||
return f"01JAXY{next(_RIDS):020d}"
|
||||
|
||||
|
||||
def fake_claude(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, **script: list[dict[str, Any]]
|
||||
) -> Path:
|
||||
"""Point ``LEETCODE_CLAUDE_BIN`` at the scripted fake; its directory."""
|
||||
root = tmp_path / "fake-claude"
|
||||
root.mkdir(exist_ok=True)
|
||||
(root / "calls.jsonl").write_text("", encoding="utf-8")
|
||||
(root / "script.json").write_text(json.dumps(script), encoding="utf-8")
|
||||
monkeypatch.setenv("LEETCODE_CLAUDE_BIN", str(FAKE_CLAUDE))
|
||||
monkeypatch.setenv("FAKE_CLAUDE_DIR", str(root))
|
||||
return root
|
||||
|
||||
|
||||
def calls(root: Path) -> list[dict[str, Any]]:
|
||||
"""Every call the fake claude answered."""
|
||||
lines = (root / "calls.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
return [json.loads(line) for line in lines]
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeProbe:
|
||||
"""``fetch_recent_ac`` as a switch: unusable, nothing, or an Accepted."""
|
||||
|
||||
status: str = "nothing"
|
||||
timestamp_s: int = 0
|
||||
calls: int = 0
|
||||
|
||||
def __call__(self) -> SolveProbe:
|
||||
self.calls += 1
|
||||
if self.status == "unusable":
|
||||
return SolveProbe(ProbeStatus.UNVERIFIABLE, (), "session expired")
|
||||
rows = (
|
||||
(
|
||||
AcSubmission(
|
||||
"1", "Missing Number", "missing-number", self.timestamp_s, "js"
|
||||
),
|
||||
)
|
||||
if self.status == "accepted"
|
||||
else ()
|
||||
)
|
||||
return SolveProbe(ProbeStatus.OK, rows, "ok")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Clock:
|
||||
"""A monotonic clock the test moves (the probe's minute)."""
|
||||
|
||||
now: float = 1000.0
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
|
||||
@dataclass
|
||||
class Rig:
|
||||
"""One relay and what drives it."""
|
||||
|
||||
relay: Relay
|
||||
probe: FakeProbe
|
||||
clock: Clock
|
||||
sessions: Path
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def make_rig(tmp_path: Path, *, start: bool = True, **deps: Any) -> Rig:
|
||||
"""A relay on port 0 with fake fetch/probe and dry-run cards.
|
||||
|
||||
``heartbeat_s`` / ``rescan_s`` / ``pace_s`` go to the `Relay`, the rest to `Deps`.
|
||||
"""
|
||||
timing = {
|
||||
k: deps.pop(k) for k in ("heartbeat_s", "rescan_s", "pace_s") if k in deps
|
||||
}
|
||||
probe, clock = FakeProbe(), Clock()
|
||||
base: dict[str, Any] = {
|
||||
"state_dir": tmp_path / "relay",
|
||||
"fetch": lambda _slug: ("Missing Number", MISSING_NUMBER),
|
||||
"probe": AcceptedProbe(probe, monotonic=clock),
|
||||
"cards_dry_run": True,
|
||||
}
|
||||
relay = Relay(
|
||||
Deps(**(base | deps)), port=0, tick_s=0.02, **({"pace_s": 0.0} | timing)
|
||||
)
|
||||
if start:
|
||||
relay.start()
|
||||
return Rig(relay, probe, clock, tmp_path / "sessions")
|
||||
|
||||
|
||||
def http(rig: Rig, method: str, path: str, payload: object = None) -> tuple[int, Any]:
|
||||
"""One loopback call; ``(status, decoded body)``."""
|
||||
body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode()
|
||||
headers = {} if body is None else {"Content-Type": "application/json"}
|
||||
conn = HTTPConnection("127.0.0.1", rig.relay.port, timeout=10)
|
||||
try:
|
||||
conn.request(method, path, body=body, headers=headers)
|
||||
resp = conn.getresponse()
|
||||
return resp.status, json.loads(resp.read())
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def call(rig: Rig, kind: str, **body: object) -> dict[str, Any]:
|
||||
"""Send one request and poll until its result (what the app does)."""
|
||||
rid_ = rid()
|
||||
status, got = http(
|
||||
rig,
|
||||
"POST",
|
||||
"/coach/requests",
|
||||
{"rid": rid_, "kind": kind, "dev": DEV, "body": body},
|
||||
)
|
||||
if status == 200:
|
||||
return dict(got)
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
_status, polled = http(rig, "GET", f"/coach/requests/{rid_}")
|
||||
if polled["result"] is not None:
|
||||
return dict(polled["result"])
|
||||
time.sleep(0.01)
|
||||
msg = f"no result for {kind}"
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
def wait_for(check: Any, timeout: float = 5.0) -> Any:
|
||||
"""Poll `check` until it returns something truthy."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if got := check():
|
||||
return got
|
||||
time.sleep(0.01)
|
||||
msg = "condition never held"
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
def guard_digest() -> str:
|
||||
"""A hash of every file under the (redirected) guard data dir."""
|
||||
digest = hashlib.sha256()
|
||||
root = Path(_constants.DATA_DIR)
|
||||
for path in sorted(root.rglob("*")):
|
||||
digest.update(str(path.relative_to(root)).encode())
|
||||
if path.is_file():
|
||||
digest.update(path.read_bytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def seed_guard() -> None:
|
||||
"""Put a ledger and caches where the guard would read them."""
|
||||
root = Path(_constants.DATA_DIR)
|
||||
(root / "ledger.json").write_text(
|
||||
'{"entries": [{"k": "credit:1"}]}', encoding="utf-8"
|
||||
)
|
||||
(root / "pool_cache.json").write_text("{}", encoding="utf-8")
|
||||
172
leetcode_coach/leetcode_coach/tests/test_app_events.py
Normal file
172
leetcode_coach/leetcode_coach/tests/test_app_events.py
Normal file
@ -0,0 +1,172 @@
|
||||
"""The coach modules' side of the relay contract (DOCS-coach-relay.md 2 and 5).
|
||||
|
||||
New session-log events, sessions split at ``session_start``, the step-2
|
||||
force that must open code, and the env/timeout seams the relay calls with.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach import (
|
||||
claude_cli,
|
||||
local_backend,
|
||||
phase,
|
||||
req_check,
|
||||
session_log,
|
||||
toolbox,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
NEW_EVENTS = ("session_start", "grade", "chat", "session_end")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event", NEW_EVENTS)
|
||||
def test_the_app_events_are_part_of_the_contract(tmp_path: Path, event: str) -> None:
|
||||
path = tmp_path / "log.jsonl"
|
||||
session_log.append(path, event, sid="20261023-170448-5b71")
|
||||
assert session_log.read(path)[0]["event"] == event
|
||||
|
||||
|
||||
def test_the_docstring_allows_new_names_only_with_the_reader() -> None:
|
||||
doc = " ".join((session_log.__doc__ or "").split())
|
||||
assert "never as a new event name" not in doc
|
||||
assert "new names only together with the reader" in doc
|
||||
|
||||
|
||||
def test_current_is_everything_after_the_last_session_start() -> None:
|
||||
cli = [{"event": "req"}, {"event": "code"}]
|
||||
assert session_log.current(cli) == cli
|
||||
app = [
|
||||
{"event": "session_start", "sid": "a"},
|
||||
{"event": "code"},
|
||||
{"event": "session_start", "sid": "b"},
|
||||
{"event": "rung", "rung": "H1"},
|
||||
]
|
||||
assert session_log.current([*cli, *app]) == app[2:]
|
||||
|
||||
|
||||
def test_code_from_an_earlier_session_does_not_count(tmp_path: Path) -> None:
|
||||
log = tmp_path / "log.jsonl"
|
||||
session_log.append(log, "code")
|
||||
assert session_log.has_code(log)
|
||||
session_log.append(log, "session_start", sid="s")
|
||||
assert not session_log.has_code(log)
|
||||
|
||||
|
||||
def _grade(step: int, grade: str, *, forced: bool = False) -> dict[str, Any]:
|
||||
return {"event": "grade", "step": step, "grade": grade, "forced": forced}
|
||||
|
||||
|
||||
def test_a_forced_step_two_opens_code() -> None:
|
||||
events = [{"event": "req_check", "verdict": "incomplete"}, _grade(2, "fail")]
|
||||
assert not phase.req_state(events).open
|
||||
forced = phase.req_state([*events, _grade(2, "fail", forced=True)])
|
||||
assert forced.open
|
||||
assert forced.forced
|
||||
assert not forced.accepted
|
||||
assert not phase.req_state([_grade(1, "pass", forced=True)]).open
|
||||
|
||||
|
||||
def test_step_two_attempts_count_incomplete_lists_and_fail_grades() -> None:
|
||||
events = [
|
||||
{"event": "req_check", "verdict": "incomplete"},
|
||||
_grade(2, "partial"),
|
||||
_grade(2, "fail"),
|
||||
_grade(1, "fail"),
|
||||
{"event": "req_check", "verdict": "incomplete"},
|
||||
]
|
||||
assert phase.step2_attempts(events) == 3
|
||||
|
||||
|
||||
def _gate(log: Path, answers: list[bool] | None = None) -> phase.PhaseGate:
|
||||
asked: list[str] = []
|
||||
|
||||
def ask(_args: list[str], prompt: str, **_kw: object) -> dict[str, Any]:
|
||||
asked.append(prompt)
|
||||
return {"result": json.dumps({"code": (answers or [False]).pop(0)})}
|
||||
|
||||
gate = phase.PhaseGate(log, toolbox.load_safe(), lambda: "statement", ask)
|
||||
gate.known = "statement"
|
||||
return gate
|
||||
|
||||
|
||||
def test_screen_takes_a_verdict_already_reached(tmp_path: Path) -> None:
|
||||
log = tmp_path / "log.jsonl"
|
||||
gate = _gate(log)
|
||||
session_log.append(log, "grade", step=2, grade="pass", forced=True)
|
||||
sent, note = gate.screen("a = b", is_code=True)
|
||||
assert sent == "a = b"
|
||||
assert note == ""
|
||||
assert session_log.read(log)[-1]["event"] == "code"
|
||||
assert gate.screen("x = y", is_code=False) == ("x = y", "")
|
||||
assert [e["event"] for e in session_log.read(log)].count("code") == 1
|
||||
|
||||
|
||||
def test_screen_judges_only_the_current_session(tmp_path: Path) -> None:
|
||||
log = tmp_path / "log.jsonl"
|
||||
session_log.append(log, "req_check", verdict="pass", missing=[])
|
||||
session_log.append(log, "session_start", sid="s")
|
||||
sent, note = _gate(log).screen("const x = 1;")
|
||||
assert sent is None
|
||||
assert "requirement list first" in note
|
||||
|
||||
|
||||
def test_property_line_is_public() -> None:
|
||||
box = toolbox.load_safe()
|
||||
prop = next(iter(box.properties))
|
||||
assert phase.property_line(prop, box).startswith(prop)
|
||||
|
||||
|
||||
def test_ask_merges_the_extra_env(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
stub = tmp_path / "claude"
|
||||
stub.write_text(
|
||||
'#!/bin/sh\nprintf \'{"result": "%s|%s|%s"}\' '
|
||||
'"$LEETCODE_COACH_LOG" "$CLAUDE_TTY_FIX_OFF" "$KEPT"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
stub.chmod(0o755)
|
||||
monkeypatch.setenv("LEETCODE_CLAUDE_BIN", str(stub))
|
||||
monkeypatch.setenv("KEPT", "yes")
|
||||
reply = claude_cli.ask([], "hi", env={"LEETCODE_COACH_LOG": "/x.jsonl"})
|
||||
assert reply["result"] == "/x.jsonl|1|yes"
|
||||
|
||||
|
||||
def test_ask_models_is_public_and_bounds_the_local_call(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
seen: list[float] = []
|
||||
|
||||
def local(messages: list[dict[str, str]], *, timeout: float) -> object:
|
||||
seen.append(timeout)
|
||||
return local_backend.LocalReply('{"ok": 1}', 1.0, 1)
|
||||
|
||||
monkeypatch.setattr(local_backend, "chat", local)
|
||||
|
||||
def limit(*_a: object, **_kw: object) -> dict[str, Any]:
|
||||
return {"result": "You've hit your session limit"}
|
||||
|
||||
got = req_check.ask_models("sys", "p", lambda o: o, limit)
|
||||
assert got == ({"ok": 1}, "ollama")
|
||||
assert seen == [req_check.LOCAL_TIMEOUT_S]
|
||||
assert req_check.LOCAL_TIMEOUT_S + req_check.HAIKU_TIMEOUT_S < 280
|
||||
|
||||
|
||||
def test_local_chat_takes_a_timeout() -> None:
|
||||
timeouts: list[float] = []
|
||||
|
||||
def post(_url: str, _payload: object, timeout: float) -> dict[str, Any]:
|
||||
timeouts.append(timeout)
|
||||
return {"message": {"content": "hi"}}
|
||||
|
||||
reply = local_backend.chat([{"role": "user", "content": "x"}], post=post, timeout=7)
|
||||
assert reply.text == "hi"
|
||||
local_backend.chat([{"role": "user", "content": "x"}], post=post)
|
||||
assert timeouts == [7, local_backend.REQUEST_TIMEOUT_S]
|
||||
99
leetcode_coach/leetcode_coach/tests/test_relay_boundary.py
Normal file
99
leetcode_coach/leetcode_coach/tests/test_relay_boundary.py
Normal file
@ -0,0 +1,99 @@
|
||||
"""No relay route reaches the ledger (TODO-coach-ui.md decision 2, contract 4).
|
||||
|
||||
Walks the import closure of every ``leetcode_coach.relay`` module by AST --
|
||||
through ``leetcode_coach`` and ``leetcode_guard`` alike, ``TYPE_CHECKING``
|
||||
imports included -- and holds the guard side to an allowlist: the read-only
|
||||
submission probe, its client, and the Firebase sign-in. A credit still comes
|
||||
only from a LeetCode Accepted the harvester sees.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
import leetcode_coach
|
||||
|
||||
ROOT = Path(leetcode_coach.__file__).resolve().parent.parent
|
||||
ALLOWED = {
|
||||
"_submissions",
|
||||
"_queries",
|
||||
"_settings",
|
||||
"_auth",
|
||||
"_constants",
|
||||
"_leetcode",
|
||||
"_throttle",
|
||||
"_snapshot_auth",
|
||||
}
|
||||
DENIED = re.compile(
|
||||
r"_ledger.*|_balance|_debt|_harvest.*|_gate.*|_sync.*|_lock.*|_breakglass"
|
||||
r"|_escape.*|_snapshot_push"
|
||||
)
|
||||
PACKAGES = ("leetcode_coach", "leetcode_guard")
|
||||
|
||||
|
||||
def _file(module: str) -> Path | None:
|
||||
base = ROOT.joinpath(*module.split("."))
|
||||
for path in (base.with_suffix(".py"), base / "__init__.py"):
|
||||
if path.is_file():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def _imports(path: Path) -> set[str]:
|
||||
found: set[str] = set()
|
||||
for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))):
|
||||
if isinstance(node, ast.Import):
|
||||
found |= {alias.name for alias in node.names}
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
found.add(node.module)
|
||||
found |= {f"{node.module}.{alias.name}" for alias in node.names}
|
||||
return {m for m in found if m.split(".")[0] in PACKAGES and _file(m)}
|
||||
|
||||
|
||||
def closure() -> set[str]:
|
||||
"""Every first-party module the relay package can load."""
|
||||
start = {
|
||||
f"leetcode_coach.relay.{p.stem}"
|
||||
for p in (ROOT / "leetcode_coach" / "relay").glob("*.py")
|
||||
}
|
||||
seen: set[str] = set()
|
||||
todo = sorted(start)
|
||||
while todo:
|
||||
module = todo.pop()
|
||||
if module in seen:
|
||||
continue
|
||||
seen.add(module)
|
||||
path = _file(module)
|
||||
assert path is not None, module
|
||||
todo += sorted(_imports(path) - seen)
|
||||
return seen
|
||||
|
||||
|
||||
def test_the_relay_reaches_only_the_read_only_guard_modules() -> None:
|
||||
guard = {m.split(".")[1] for m in closure() if m.startswith("leetcode_guard.")}
|
||||
assert guard <= ALLOWED, guard - ALLOWED
|
||||
assert not [m for m in guard if DENIED.fullmatch(m)]
|
||||
assert "_submissions" in guard # the walk does see the guard side
|
||||
|
||||
|
||||
def test_the_walk_would_catch_a_ledger_import(tmp_path: Path) -> None:
|
||||
probe = tmp_path / "x.py"
|
||||
probe.write_text(
|
||||
"from leetcode_guard import _ledger\nimport leetcode_guard._gate\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert _imports(probe) >= {"leetcode_guard._ledger", "leetcode_guard._gate"}
|
||||
assert DENIED.fullmatch("_ledger_io")
|
||||
assert DENIED.fullmatch("_harvest_run")
|
||||
|
||||
|
||||
def test_no_relay_module_spawns_anything_but_claude_and_node() -> None:
|
||||
relay = ROOT / "leetcode_coach" / "relay"
|
||||
spawning = {
|
||||
p.name
|
||||
for p in relay.glob("*.py")
|
||||
if re.search(r"\bsubprocess\b|os\.system|os\.exec|Popen", p.read_text())
|
||||
}
|
||||
assert spawning == {"runcode.py"} # claude runs via claude_cli.ask
|
||||
187
leetcode_coach/leetcode_coach/tests/test_relay_coach.py
Normal file
187
leetcode_coach/leetcode_coach/tests/test_relay_coach.py
Normal file
@ -0,0 +1,187 @@
|
||||
"""The session backend: lifecycle, chat, hand-over, restart (contract 2-3)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
import urllib.error
|
||||
|
||||
from leetcode_coach import local_backend, session_log
|
||||
from leetcode_coach.relay import session as session_mod
|
||||
from leetcode_coach.tests._relay_direct import FakeAsk, make_coach, req, started
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
LIMIT = "You've hit your session limit · resets 3pm (Europe/Warsaw)"
|
||||
|
||||
|
||||
def test_no_session_yet_is_a_known_empty_state(tmp_path: Path) -> None:
|
||||
coach = make_coach(tmp_path, FakeAsk([]))
|
||||
state = coach.state()
|
||||
assert state == {
|
||||
"schema": 1,
|
||||
"rev": 1,
|
||||
"step": None,
|
||||
"rung": None,
|
||||
"revealed": False,
|
||||
"chat_count": 0,
|
||||
"session": None,
|
||||
"steps": [],
|
||||
}
|
||||
assert coach.chat_after(None) == {}
|
||||
assert (
|
||||
coach.handle(req("hint", session="20261023-170448-5b71"))["code"] == "not_live"
|
||||
)
|
||||
|
||||
|
||||
def test_end_then_a_fresh_session_clears_the_chat(tmp_path: Path) -> None:
|
||||
ask = FakeAsk([{"result": "One?"}, {"result": "Two?"}])
|
||||
coach = make_coach(tmp_path, ask)
|
||||
sid = started(coach)
|
||||
assert coach.handle(req("session_end", session=sid))["data"] == {}
|
||||
state = coach.state()
|
||||
assert state["session"]["end_reason"] == "ended"
|
||||
assert state["step"] is None
|
||||
assert coach.handle(req("chat", session=sid, text="hi"))["code"] == "not_live"
|
||||
other = started(coach)
|
||||
assert other != sid
|
||||
assert list(coach.chat_after(None)) == ["m0001"]
|
||||
assert coach.chat_after(None)["m0001"]["text"] == "Two?"
|
||||
events = session_log.read(coach.session.log)
|
||||
assert [e["event"] for e in events].count("session_start") == 2
|
||||
|
||||
|
||||
def test_only_answer_key_problems_and_a_failed_opener_logs_nothing(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
ask = FakeAsk([{"result": "x", "session_id": ""}])
|
||||
coach = make_coach(tmp_path, ask, fetch=lambda _slug: None)
|
||||
assert coach.handle(req("session_start", slug="two-sum"))["code"] == "no_answer_key"
|
||||
failed = coach.handle(req("session_start", slug="missing-number"))
|
||||
assert failed["code"] == "model_error"
|
||||
assert failed["retry"] is True
|
||||
assert coach.session is None
|
||||
assert not list(session_log.sessions_dir().glob("*.jsonl"))
|
||||
|
||||
|
||||
def test_a_usage_limit_hands_over_to_the_local_model(tmp_path: Path) -> None:
|
||||
seen: list[Any] = []
|
||||
|
||||
def local(
|
||||
messages: list[dict[str, str]], *, timeout: float
|
||||
) -> local_backend.LocalReply:
|
||||
seen.append((messages[-1]["content"], timeout))
|
||||
return local_backend.LocalReply("What does a pair do?", 1.0, 5)
|
||||
|
||||
ask = FakeAsk([{"result": "Start?"}, {"result": LIMIT}])
|
||||
coach = make_coach(tmp_path, ask, local_chat=local)
|
||||
sid = started(coach)
|
||||
reply = coach.handle(req("chat", session=sid, text="I am stuck"))
|
||||
assert reply["ok"]
|
||||
assert coach.session.local
|
||||
again = coach.handle(req("chat", session=sid, text="still stuck"))
|
||||
assert again["ok"]
|
||||
assert len(ask.prompts()) == 2 # the second turn never asked claude
|
||||
assert seen[-1][1] == 240.0
|
||||
assert coach.chat_after("m0003")["m0005"]["text"] == "What does a pair do?"
|
||||
|
||||
|
||||
def test_quota_and_local_both_down_is_claude_unavailable(tmp_path: Path) -> None:
|
||||
def down(_messages: object, **_kw: object) -> local_backend.LocalReply:
|
||||
msg = "refused"
|
||||
raise urllib.error.URLError(msg)
|
||||
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": LIMIT}]), local_chat=down)
|
||||
failed = coach.handle(req("session_start", slug="missing-number"))
|
||||
assert failed["code"] == "claude_unavailable"
|
||||
assert failed["resets_at"] == "3pm (Europe/Warsaw)"
|
||||
assert failed["reason"] == LIMIT
|
||||
|
||||
|
||||
def test_reveal_goes_out_bare_and_ends_the_session(tmp_path: Path) -> None:
|
||||
ask = FakeAsk([{"result": "Hi?"}, {"result": "Here is the idea."}])
|
||||
coach = (
|
||||
make_coach(tmp_path, ask, haiku=None) if False else make_coach(tmp_path, ask)
|
||||
)
|
||||
sid = started(coach)
|
||||
coach.session.notes.append("[app: pending]")
|
||||
got = coach.handle(req("reveal", session=sid))
|
||||
assert got["data"] == {"msg": "m0003"}
|
||||
assert ask.prompts()[-1] == "reveal"
|
||||
assert coach.state()["revealed"] is True
|
||||
assert coach.state()["session"]["end_reason"] == "reveal"
|
||||
|
||||
|
||||
def test_chat_req_lists_and_ambiguous_text(tmp_path: Path) -> None:
|
||||
haiku = [
|
||||
{"result": json.dumps({"verdict": "pass", "missing": []})},
|
||||
{"result": json.dumps({"code": False})},
|
||||
]
|
||||
ask = FakeAsk([{"result": "Hi?"}, {"result": "Ok?"}, {"result": "Sure?"}], haiku)
|
||||
coach = make_coach(tmp_path, ask)
|
||||
sid = started(coach)
|
||||
assert coach.handle(req("chat", session=sid, text="req: self-inverse"))["ok"]
|
||||
assert any(v["kind"] == "grade" for v in coach.chat_after(None).values())
|
||||
assert coach.handle(req("chat", session=sid, text="so a = b then"))["ok"]
|
||||
assert ask.prompts("haiku")[1] == "so a = b then"
|
||||
|
||||
|
||||
def test_a_screen_refusal_is_phase_locked(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": "Hi?"}]))
|
||||
sid = started(coach)
|
||||
monkeypatch.setattr(coach.session.gate, "screen", lambda *_a, **_k: (None, "held"))
|
||||
got = coach.handle(req("chat", session=sid, text="just words"))
|
||||
assert got["code"] == "phase_locked"
|
||||
assert got["note"] == "held"
|
||||
|
||||
|
||||
def test_a_restart_rebuilds_the_session_from_the_log(tmp_path: Path) -> None:
|
||||
ask = FakeAsk([{"result": "Hi? <!-- rung:H0 -->"}])
|
||||
coach = make_coach(tmp_path, ask)
|
||||
sid = started(coach)
|
||||
haiku = [{"result": json.dumps({"grade": "partial", "feedback": "More."})}]
|
||||
coach.deps.ask = FakeAsk([], haiku)
|
||||
coach.session.gate.ask = coach.deps.ask
|
||||
assert coach.handle(req("grade_step", session=sid, step=1, answer="n"))["ok"]
|
||||
rev = coach.rev
|
||||
again = make_coach(tmp_path, FakeAsk([]))
|
||||
assert again.session is not None
|
||||
assert again.session.sid == sid
|
||||
assert again.session.claude_id == "s1"
|
||||
assert again.rev > rev
|
||||
kinds = [(v["role"], v["kind"]) for v in again.chat_after(None).values()]
|
||||
assert kinds == [("coach", "chat"), ("system", "grade")]
|
||||
assert again.state()["steps"][0]["attempts"] == 1
|
||||
assert again.state()["rung"] == "H0"
|
||||
|
||||
|
||||
def test_a_broken_or_stale_pointer_restores_nothing(tmp_path: Path) -> None:
|
||||
live = tmp_path / "relay" / "live.json"
|
||||
live.parent.mkdir(parents=True)
|
||||
live.write_text("{", encoding="utf-8")
|
||||
assert make_coach(tmp_path, FakeAsk([])).session is None
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": "Hi?"}]))
|
||||
started(coach)
|
||||
record = json.loads(live.read_text(encoding="utf-8"))
|
||||
session_log.append(coach.session.log, "session_start", sid="20991231-000000-ffff")
|
||||
assert make_coach(tmp_path, FakeAsk([])).session is None
|
||||
record["log"] = str(tmp_path / "gone.jsonl")
|
||||
live.write_text(json.dumps(record), encoding="utf-8")
|
||||
assert make_coach(tmp_path, FakeAsk([])).session is None
|
||||
|
||||
|
||||
def test_event_times_fall_back_to_ts(caplog: pytest.LogCaptureFixture) -> None:
|
||||
assert session_mod._ms({"at": 5}) == 5
|
||||
assert session_mod._ms({"ts": "2026-10-23T17:04:48+02:00"}) == 1792767888000
|
||||
assert session_mod._ms({"ts": "yesterday"}) == 0
|
||||
assert "usable time" in caplog.text
|
||||
forced = {"step": 2, "forced": True, "attempt": 3}
|
||||
assert session_mod.grade_line(forced) == (
|
||||
"grade",
|
||||
"Step 2 (requirements): continued after 3 tries.",
|
||||
)
|
||||
52
leetcode_coach/leetcode_coach/tests/test_relay_e2e.py
Normal file
52
leetcode_coach/leetcode_coach/tests/test_relay_e2e.py
Normal file
@ -0,0 +1,52 @@
|
||||
"""TODO-coach-ui.md "Done", desktop path: the scripted session over loopback.
|
||||
|
||||
The flow and its assertions live in ``_relay_flow``; here it runs over the
|
||||
real HTTP API on port 0, with the guard's paths redirected (so the data dir
|
||||
check cannot race the live harvester).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.tests import _relay_flow as flow
|
||||
from leetcode_coach.tests import _relay_support as rs
|
||||
from leetcode_guard.tests._paths_fixture import _isolate_paths
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
__all__ = ["_isolate_paths"] # the guard's paths, redirected for every test here
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rig(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[rs.Rig]:
|
||||
root = rs.fake_claude(tmp_path, monkeypatch, **flow.SCRIPT)
|
||||
rs.seed_guard()
|
||||
made = rs.make_rig(tmp_path)
|
||||
made.extra["claude"] = root
|
||||
yield made
|
||||
made.relay.stop()
|
||||
|
||||
|
||||
def desktop(rig: rs.Rig) -> flow.Ends:
|
||||
def state() -> dict[str, Any]:
|
||||
return dict(rs.http(rig, "GET", "/coach/state")[1])
|
||||
|
||||
def chat(after: str | None) -> dict[str, Any]:
|
||||
query = "" if after is None else f"?after={after}"
|
||||
return dict(rs.http(rig, "GET", f"/coach/chat{query}")[1])
|
||||
|
||||
return flow.Ends(lambda kind, **body: rs.call(rig, kind, **body), state, chat)
|
||||
|
||||
|
||||
def test_a_full_session_on_the_desktop(rig: rs.Rig) -> None:
|
||||
flow.run_flow(rig, desktop(rig))
|
||||
status, beat = rs.http(rig, "GET", "/coach/heartbeat")
|
||||
assert status == 200
|
||||
assert beat["state_rev"] == rig.relay.coach.rev
|
||||
assert beat["version"] == 1
|
||||
assert "busy" not in beat
|
||||
98
leetcode_coach/leetcode_coach/tests/test_relay_examples.py
Normal file
98
leetcode_coach/leetcode_coach/tests/test_relay_examples.py
Normal file
@ -0,0 +1,98 @@
|
||||
"""The statement's examples and function name, as run_code needs them."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay import examples
|
||||
from leetcode_coach.relay.examples import Example
|
||||
from leetcode_coach.tests._relay_statements import (
|
||||
MERGE_INTERVALS,
|
||||
MISSING_NUMBER,
|
||||
TWO_SUM,
|
||||
)
|
||||
|
||||
|
||||
def test_plain_bold_labels() -> None:
|
||||
name, found = examples.parse(MISSING_NUMBER)
|
||||
assert name == "missingNumber"
|
||||
assert found == [
|
||||
Example(1, ([3, 0, 1],), 2),
|
||||
Example(2, ([0, 1],), 2),
|
||||
Example(3, ([9, 6, 4, 2, 3, 5, 7, 0, 1],), 8),
|
||||
]
|
||||
|
||||
|
||||
def test_code_spans_and_several_arguments_in_order() -> None:
|
||||
name, found = examples.parse(TWO_SUM)
|
||||
assert name == "twoSum"
|
||||
assert found[0] == Example(1, ([2, 7, 11, 15], 9), [0, 1])
|
||||
assert len(found) == 3
|
||||
|
||||
|
||||
def test_fenced_nested_arrays() -> None:
|
||||
name, found = examples.parse(MERGE_INTERVALS)
|
||||
assert name == "merge"
|
||||
assert found[1] == Example(2, ([[1, 4], [4, 5]],), [[1, 5]])
|
||||
|
||||
|
||||
def test_strings_with_commas_and_literals() -> None:
|
||||
text = (
|
||||
'**Input:** s = "a,b", k = [1,[2,3]], flag = true, none = null\n'
|
||||
"**Output:** false\n"
|
||||
"function solve(s, k, flag, none) {}\n"
|
||||
)
|
||||
name, found = examples.parse(text)
|
||||
assert name == "solve"
|
||||
assert found == [Example(1, ("a,b", [1, [2, 3]], True, None), False)]
|
||||
|
||||
|
||||
def test_escaped_quote_inside_a_string() -> None:
|
||||
text = '**Input:** s = "a\\",b"\n**Output:** "x"\n'
|
||||
assert examples.parse(text)[1] == [Example(1, ('a",b',), "x")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("starter", "name"),
|
||||
[
|
||||
("const best = (nums) => 0;", "best"),
|
||||
("let best = function (nums) {};", "best"),
|
||||
("function best(nums) {}", "best"),
|
||||
("class Solution {}", None),
|
||||
],
|
||||
)
|
||||
def test_function_name_shapes(starter: str, name: str | None) -> None:
|
||||
assert (
|
||||
examples.parse(f"## Starter code (javascript)\n```\n{starter}\n```\n")[0]
|
||||
== name
|
||||
)
|
||||
|
||||
|
||||
def test_an_example_whose_value_is_not_json_is_skipped(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
text = (
|
||||
"**Input:** root = [1,null,2]\n**Output:** [1,2]\n"
|
||||
"**Input:** head = 1->2\n**Output:** 2\n"
|
||||
"**Input:** n = 3\n**Output:** nope\n"
|
||||
"**Input:** 3\n**Output:** 3\n"
|
||||
)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
_, found = examples.parse(text)
|
||||
assert found == [Example(1, ([1, None, 2],), [1, 2])]
|
||||
assert "skipped" in caplog.text
|
||||
|
||||
|
||||
def test_output_without_input_and_hint_section_are_ignored() -> None:
|
||||
text = (
|
||||
"**Output:** 4\n**Input:** n = 1\n**Output:** 1\n"
|
||||
"## Site hints (do NOT show to the user unasked)\n"
|
||||
"- **Input:** n = 9\n**Output:** 9\n"
|
||||
)
|
||||
assert examples.parse(text)[1] == [Example(1, (1,), 1)]
|
||||
|
||||
|
||||
def test_nothing_to_find() -> None:
|
||||
assert examples.parse("") == (None, [])
|
||||
205
leetcode_coach/leetcode_coach/tests/test_relay_grading.py
Normal file
205
leetcode_coach/leetcode_coach/tests/test_relay_grading.py
Normal file
@ -0,0 +1,205 @@
|
||||
"""grade_step's branches: fails without a model, fail-open, force, cards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from leetcode_coach import answer_key, card_rules, make_cards, session_log
|
||||
from leetcode_coach.relay import cards, graders, grading, turns
|
||||
from leetcode_coach.tests._relay_direct import FakeAsk, grade, make_coach, req, started
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay.coach import Coach
|
||||
|
||||
|
||||
def _verdict(grade_: str, feedback: str = "") -> dict[str, str]:
|
||||
return {"result": json.dumps({"grade": grade_, "feedback": feedback})}
|
||||
|
||||
|
||||
def _check(verdict: str, *missing: str) -> dict[str, str]:
|
||||
return {"result": json.dumps({"verdict": verdict, "missing": list(missing)})}
|
||||
|
||||
|
||||
def _coach(tmp_path: Path, *haiku: dict[str, Any], **deps: Any) -> tuple[Coach, str]:
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": "Hi?"}] * 3, list(haiku)), **deps)
|
||||
return coach, started(coach)
|
||||
|
||||
|
||||
def test_step_two_fails_code_and_tool_names_without_asking(tmp_path: Path) -> None:
|
||||
coach, sid = _coach(tmp_path, _verdict("pass"), _check("incomplete", "commutative"))
|
||||
grade(coach, sid, 1, answer="n is big")
|
||||
code = grade(coach, sid, 2, answer="req: x ^= y;")["data"]
|
||||
assert (code["grade"], code["feedback"]) == ("fail", grading.READS_AS_CODE)
|
||||
named = grade(coach, sid, 2, answer="use XOR")["data"]
|
||||
assert (named["grade"], named["feedback"], named["attempt"]) == (
|
||||
"fail",
|
||||
grading.NAMES_A_TOOL,
|
||||
2,
|
||||
)
|
||||
third = grade(coach, sid, 2, answer="order does not matter")["data"]
|
||||
assert third["grade"] == "partial"
|
||||
assert third["can_force"] is True
|
||||
events = session_log.read(coach.session.log)
|
||||
assert [e["event"] for e in events].count("req_check") == 1
|
||||
forced = grade(coach, sid, 2, force=True)["data"]
|
||||
assert forced == {"step": 2, "forced": True, "next_step": 3}
|
||||
assert coach.state()["step"] == 3
|
||||
|
||||
|
||||
def test_graders_down_pass_unverified(tmp_path: Path) -> None:
|
||||
coach, sid = _coach(tmp_path, {"result": "no json"}, {"result": "nope"})
|
||||
one = grade(coach, sid, 1, answer="n is big")["data"]
|
||||
assert (one["grade"], one["via"], one["feedback"]) == (
|
||||
"pass",
|
||||
"unverified",
|
||||
graders.UNVERIFIED,
|
||||
)
|
||||
two = grade(coach, sid, 2, answer="self-inverse")["data"]
|
||||
assert (two["grade"], two["via"]) == ("pass", "unverified")
|
||||
|
||||
|
||||
def test_feedback_that_names_a_tool_or_quotes_the_key_is_replaced(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
rule = answer_key.load_answers()["missing-number"].design_rule
|
||||
coach, sid = _coach(
|
||||
tmp_path,
|
||||
_verdict("fail", "Think about XOR."),
|
||||
_verdict("partial", "Hint: " + rule),
|
||||
)
|
||||
first = grade(coach, sid, 1, answer="n")["data"]
|
||||
assert first["feedback"] == graders.CONSTRAINT_HINT
|
||||
second = grade(coach, sid, 1, answer="n")["data"]
|
||||
assert second["feedback"] == graders.CONSTRAINT_HINT
|
||||
assert rule not in json.dumps(coach.chat_after(None))
|
||||
|
||||
|
||||
def test_locks_done_and_force_rules(tmp_path: Path) -> None:
|
||||
coach, sid = _coach(tmp_path, _verdict("partial", "More."), _verdict("pass"))
|
||||
assert grade(coach, sid, 3, picks=["XOR"])["code"] == "step_locked"
|
||||
grade(coach, sid, 1, answer="n")
|
||||
assert grade(coach, sid, 1, force=True)["code"] == "cannot_force"
|
||||
grade(coach, sid, 1, answer="n")
|
||||
assert grade(coach, sid, 1, answer="n")["code"] == "step_done"
|
||||
assert grade(coach, sid, 1, force=True)["code"] == "step_done"
|
||||
|
||||
|
||||
def _to_step(coach: Coach, sid: str, n: int) -> None:
|
||||
"""Force (or pass) every step before `n`, with no model."""
|
||||
for step in range(1, n):
|
||||
if step == 3:
|
||||
grade(coach, sid, 3, picks=["XOR"])
|
||||
elif step == 4:
|
||||
session = coach.session
|
||||
session.log_event(
|
||||
"grade", step=4, grade="pass", forced=False, via="accepted"
|
||||
)
|
||||
else:
|
||||
for _ in range(3):
|
||||
grade(coach, sid, step, answer="req: x ^= y;" if step == 2 else "n")
|
||||
grade(coach, sid, step, force=True)
|
||||
|
||||
|
||||
def test_unknown_picks_are_bad_requests(tmp_path: Path) -> None:
|
||||
coach, sid = _coach(tmp_path)
|
||||
_to_step(coach, sid, 3)
|
||||
assert grade(coach, sid, 3, picks=["telepathy"])["code"] == "bad_request"
|
||||
|
||||
|
||||
def test_forcing_the_post_mortem_ends_without_cards(tmp_path: Path) -> None:
|
||||
coach, sid = _coach(tmp_path, *[_verdict("fail", "No.")] * 6)
|
||||
_to_step(coach, sid, 5)
|
||||
for _ in range(3):
|
||||
grade(coach, sid, 5, answer="dunno")
|
||||
forced = grade(coach, sid, 5, force=True)["data"]
|
||||
assert forced == {"step": 5, "forced": True, "next_step": None}
|
||||
assert coach.state()["session"]["end_reason"] == "done"
|
||||
|
||||
|
||||
def test_a_real_delivery_logs_the_post_mortem(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(make_cards, "OUT_DIR", tmp_path / "learn")
|
||||
monkeypatch.setattr(make_cards.anki_connect, "available", lambda: False)
|
||||
coach, sid = _coach(tmp_path, _verdict("pass"), cards_dry_run=False)
|
||||
_to_step(coach, sid, 5)
|
||||
done_ = grade(coach, sid, 5, answer="the pairs cancel")
|
||||
assert done_["ok"], done_
|
||||
done = done_["data"]
|
||||
assert done["cards"] == {"built": 1, "deck": "LeetCode", "error": None}
|
||||
assert (tmp_path / "learn" / "cards.tsv").is_file()
|
||||
assert session_log.read(coach.session.log)[-2]["event"] == "postmortem"
|
||||
|
||||
|
||||
def test_card_failures_only_fill_error(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
coach, _sid = _coach(tmp_path)
|
||||
session = coach.session
|
||||
assert cards.build(session, ("nope",))["error"].startswith("no cards")
|
||||
monkeypatch.setattr(card_rules, "validate", lambda *_a: ["too long"])
|
||||
assert cards.build(session, ("XOR",))["error"] == "cards rejected: too long"
|
||||
monkeypatch.setattr(card_rules, "validate", lambda *_a: [])
|
||||
|
||||
def boom(*_a: object, **_k: object) -> str:
|
||||
msg = "anki on fire"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
monkeypatch.setattr(make_cards, "deliver", boom)
|
||||
assert cards.build(session, ("XOR",))["error"] == "delivery failed: anki on fire"
|
||||
|
||||
|
||||
def test_run_code_shows_its_error(tmp_path: Path) -> None:
|
||||
ran = {"examples": [], "error": "timeout"}
|
||||
coach, sid = _coach(tmp_path, run_code=lambda _c, _s: ran, fetch=lambda _s: None)
|
||||
_to_step(coach, sid, 3)
|
||||
assert coach.handle(req("run_code", session=sid, code="while(1){}"))["data"] == ran
|
||||
last = list(coach.chat_after(None).values())[-1]
|
||||
assert last == {
|
||||
"role": "system",
|
||||
"kind": "run",
|
||||
"text": turns.RUN_ERRORS["timeout"],
|
||||
"at": last["at"],
|
||||
}
|
||||
|
||||
|
||||
def test_the_statement_fetch_fails_soft(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def refuse(_slug: str) -> object:
|
||||
msg = "offline"
|
||||
raise OSError(msg)
|
||||
|
||||
monkeypatch.setattr(turns.fetch_problem, "fetch", refuse)
|
||||
assert turns.fetch_statement("x") is None
|
||||
monkeypatch.setattr(turns.fetch_problem, "fetch", lambda _s: {"isPaidOnly": True})
|
||||
assert turns.fetch_statement("x") is None
|
||||
question = {
|
||||
"questionFrontendId": "1",
|
||||
"title": "T",
|
||||
"difficulty": "Easy",
|
||||
"content": "<p>Do it.</p>",
|
||||
"codeSnippets": [],
|
||||
"topicTags": [],
|
||||
"hints": [],
|
||||
}
|
||||
monkeypatch.setattr(turns.fetch_problem, "fetch", lambda _s: question)
|
||||
got = turns.fetch_statement("x")
|
||||
assert got is not None
|
||||
assert got[0] == "T"
|
||||
|
||||
|
||||
def test_the_post_mortem_grader_sees_the_key_properties() -> None:
|
||||
key = answer_key.load_answers()["missing-number"]
|
||||
seen: list[str] = []
|
||||
|
||||
def ask(_args: list[str], prompt: str, **_kw: object) -> dict[str, str]:
|
||||
seen.append(prompt)
|
||||
return {"result": json.dumps({"grade": "pass", "feedback": 5})}
|
||||
|
||||
assert graders.model_grade(5, "S", "A", key, ask).via == "unverified"
|
||||
assert "invertible" in seen[0]
|
||||
assert graders.fallback(5, key).startswith("Say which properties")
|
||||
151
leetcode_coach/leetcode_coach/tests/test_relay_lane.py
Normal file
151
leetcode_coach/leetcode_coach/tests/test_relay_lane.py
Normal file
@ -0,0 +1,151 @@
|
||||
"""The paced serial lane and the done-map (DOCS-coach-relay.md section 1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from leetcode_coach.relay import done as done_mod
|
||||
from leetcode_coach.relay import lane as lane_mod
|
||||
from leetcode_coach.relay import protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _rid(n: int) -> str:
|
||||
return f"01JAXY{n:020d}"
|
||||
|
||||
|
||||
def _req(n: int, transport: str = "desktop") -> protocol.Request:
|
||||
return protocol.Request(_rid(n), "hint", "dev", {"session": "s"}, transport)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self) -> None:
|
||||
self.ms = 1_000_000
|
||||
self.mono = 100.0
|
||||
|
||||
def now_ms(self) -> int:
|
||||
return self.ms
|
||||
|
||||
def monotonic(self) -> float:
|
||||
return self.mono
|
||||
|
||||
|
||||
def test_the_done_map_survives_a_restart_and_expires(tmp_path: Path) -> None:
|
||||
clock = Clock()
|
||||
path = tmp_path / "relay" / "requests_done.jsonl"
|
||||
done = done_mod.DoneMap(path, clock.now_ms)
|
||||
done.put(_rid(1), "phone", {"ok": True})
|
||||
done.put(_rid(2), "desktop", {"ok": False})
|
||||
again = done_mod.DoneMap(path, clock.now_ms)
|
||||
assert again.get(_rid(1)) == done_mod.Done(_rid(1), clock.ms, "phone", {"ok": True})
|
||||
clock.ms += done_mod.DAY_MS + 1
|
||||
assert again.get(_rid(2)) is None
|
||||
assert again.get(_rid(1)) is not None
|
||||
assert again.get(_rid(3)) is None
|
||||
clock.ms += done_mod.KEEP_MS
|
||||
assert again.get(_rid(1)) is None
|
||||
done_mod.DoneMap(path, clock.now_ms)
|
||||
assert path.read_text(encoding="utf-8") == ""
|
||||
|
||||
|
||||
def test_a_torn_done_map_line_is_skipped(
|
||||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
path = tmp_path / "done.jsonl"
|
||||
good = {"rid": _rid(1), "at": 5, "transport": "phone", "result": {"ok": True}}
|
||||
path.write_text(f'{json.dumps(good)}\n{{"rid": \n\n[1]\n', encoding="utf-8")
|
||||
with caplog.at_level(logging.WARNING):
|
||||
done = done_mod.DoneMap(path, lambda: 10)
|
||||
assert done.get(_rid(1)) is not None
|
||||
assert caplog.text.count("done-map line skipped") == 2
|
||||
assert len(path.read_text(encoding="utf-8").splitlines()) == 1
|
||||
|
||||
|
||||
def _lane(tmp_path: Path, handle: Any, clock: Clock, **kw: Any) -> lane_mod.Lane:
|
||||
done = done_mod.DoneMap(tmp_path / "done.jsonl", clock.now_ms)
|
||||
return lane_mod.Lane(
|
||||
handle, done, clock_ms=clock.now_ms, monotonic=clock.monotonic, **kw
|
||||
)
|
||||
|
||||
|
||||
def test_fifo_paced_and_capped(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
clock = Clock()
|
||||
lane = _lane(tmp_path, lambda r: {"ok": True, "rid": r.rid}, clock)
|
||||
assert all(lane.submit(_req(n)) is None for n in range(5))
|
||||
assert lane.submit(_req(0)) is None # a resend of a queued rid
|
||||
limited = lane.submit(_req(9))
|
||||
assert limited is not None
|
||||
assert limited["code"] == "rate_limited"
|
||||
assert limited["retry"] is True
|
||||
assert limited["retry_after_ms"] == 15000
|
||||
assert lane.result(_rid(9)) == limited
|
||||
assert lane.submit(_req(9)) == limited
|
||||
assert lane.known(_rid(4))
|
||||
assert not lane.known(_rid(7))
|
||||
stop = threading.Event()
|
||||
starts: list[str | None] = []
|
||||
lane.on_busy.append(starts.append)
|
||||
seen: list[str] = []
|
||||
lane.on_result.append(lambda req, _res: seen.append(req.rid))
|
||||
assert lane._next(stop) == _req(0)
|
||||
assert lane.busy == _rid(0)
|
||||
assert lane.submit(_req(0)) is None # running: still not re-run
|
||||
lane.busy = None
|
||||
|
||||
def advance(_timeout: float | None = None) -> bool:
|
||||
clock.mono += 3.0
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(lane._cv, "wait", advance) # the clock moves per wait
|
||||
for n in range(1, 5):
|
||||
assert lane._next(stop) == _req(n)
|
||||
lane.busy = None
|
||||
stop.set()
|
||||
assert lane._next(stop) is None
|
||||
|
||||
|
||||
def test_the_worker_runs_until_stopped_and_records_results(tmp_path: Path) -> None:
|
||||
clock = Clock()
|
||||
stop = threading.Event()
|
||||
|
||||
def handle(req: protocol.Request) -> dict[str, Any]:
|
||||
if req.rid == _rid(2):
|
||||
stop.set()
|
||||
if req.rid == _rid(1):
|
||||
message = "boom"
|
||||
raise RuntimeError(message)
|
||||
return {"ok": True}
|
||||
|
||||
lane = _lane(tmp_path, handle, clock, pace_s=0.0)
|
||||
results: list[tuple[str, dict[str, Any]]] = []
|
||||
lane.on_result.append(lambda req, res: results.append((req.rid, res)))
|
||||
busy: list[str | None] = []
|
||||
lane.on_busy.append(busy.append)
|
||||
for n in range(3):
|
||||
lane.submit(_req(n))
|
||||
lane.run(stop)
|
||||
assert [rid for rid, _ in results] == [_rid(0), _rid(1), _rid(2)]
|
||||
assert busy == [_rid(0), None, _rid(1), None, _rid(2), None]
|
||||
assert results[1][1]["code"] == "worker_error"
|
||||
assert lane.result(_rid(0)) == {"ok": True}
|
||||
assert lane.known(_rid(3)) is False
|
||||
lane.stop()
|
||||
|
||||
|
||||
def test_an_idle_lane_stops(tmp_path: Path) -> None:
|
||||
clock = Clock()
|
||||
lane = _lane(tmp_path, lambda _r: {}, clock)
|
||||
stop = threading.Event()
|
||||
worker = threading.Thread(target=lane.run, args=(stop,))
|
||||
worker.start()
|
||||
stop.set()
|
||||
lane.stop()
|
||||
worker.join(timeout=2)
|
||||
assert not worker.is_alive()
|
||||
145
leetcode_coach/leetcode_coach/tests/test_relay_limits.py
Normal file
145
leetcode_coach/leetcode_coach/tests/test_relay_limits.py
Normal file
@ -0,0 +1,145 @@
|
||||
"""Rate limit, size caps and bad requests on both paths (contract 1 and 4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay.rtdb import Rtdb
|
||||
from leetcode_coach.tests import _relay_support as rs
|
||||
from leetcode_coach.tests._fake_rtdb import FakeRtdb
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rig(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[rs.Rig]:
|
||||
rs.fake_claude(tmp_path, monkeypatch, haiku=[], coach=[])
|
||||
made = rs.make_rig(tmp_path, pace_s=100.0)
|
||||
yield made
|
||||
made.relay.stop()
|
||||
|
||||
|
||||
def _raw(
|
||||
rig: rs.Rig, method: str, path: str, body: bytes = b"", **headers: str
|
||||
) -> tuple[int, Any]:
|
||||
conn = http.client.HTTPConnection("127.0.0.1", rig.relay.port, timeout=10)
|
||||
conn.putrequest(method, path, skip_host="Host" in headers)
|
||||
for name, value in headers.items():
|
||||
conn.putheader(name, value)
|
||||
if method == "POST":
|
||||
conn.putheader("Content-Length", headers.get("Length", str(len(body))))
|
||||
conn.endheaders(body)
|
||||
resp = conn.getresponse()
|
||||
status, data = resp.status, json.loads(resp.read())
|
||||
conn.close()
|
||||
return status, data
|
||||
|
||||
|
||||
def _req(kind: str = "hint", rid: str | None = None, **body: object) -> dict[str, Any]:
|
||||
return {"rid": rid or rs.rid(), "kind": kind, "dev": rs.DEV, "body": body}
|
||||
|
||||
|
||||
def test_http_refuses_what_the_contract_refuses(rig: rs.Rig) -> None:
|
||||
json_ = {"Content-Type": "application/json"}
|
||||
big = json.dumps(_req(text="x" * 17000)).encode()
|
||||
assert _raw(rig, "POST", "/coach/requests", big, **json_)[0] == 413
|
||||
assert (
|
||||
_raw(rig, "POST", "/coach/requests", b"{}", **{"Content-Type": "text/plain"})[0]
|
||||
== 415
|
||||
)
|
||||
assert _raw(rig, "POST", "/coach/requests", b"{", **json_)[0] == 400
|
||||
assert _raw(rig, "POST", "/coach/requests", b"{}", **json_, Length="x")[0] == 400
|
||||
assert _raw(rig, "POST", "/coach/requests", b"[]", **json_)[0] == 400
|
||||
bad_rid = json.dumps(_req(rid="nope")).encode()
|
||||
assert _raw(rig, "POST", "/coach/requests", bad_rid, **json_)[0] == 400
|
||||
assert _raw(rig, "POST", "/coach/other", b"{}", **json_)[0] == 404
|
||||
assert _raw(rig, "GET", "/coach/nope")[0] == 404
|
||||
assert _raw(rig, "GET", "/coach/requests/" + rs.rid())[0] == 404
|
||||
assert _raw(rig, "GET", "/coach/requests/not-a-rid")[0] == 404
|
||||
assert _raw(rig, "GET", "/coach/state", Host="evil.example:8777")[0] == 403
|
||||
assert _raw(rig, "POST", "/coach/requests", b"{}", Host="evil.example")[0] == 403
|
||||
|
||||
|
||||
def test_a_bad_request_is_answered_and_remembered(rig: rs.Rig) -> None:
|
||||
raw = _req("rm -rf")
|
||||
status, got = rs.http(rig, "POST", "/coach/requests", raw)
|
||||
assert status == 200
|
||||
assert got["code"] == "bad_request"
|
||||
assert rs.http(rig, "POST", "/coach/requests", raw)[1] == got
|
||||
assert rs.http(rig, "GET", f"/coach/requests/{raw['rid']}")[1] == {"result": got}
|
||||
huge = _req("chat", session="20261023-170448-5b71", text="é" * 4097)
|
||||
assert rs.http(rig, "POST", "/coach/requests", huge)[1]["code"] == "bad_request"
|
||||
|
||||
|
||||
def test_more_than_five_waiting_is_rate_limited(rig: rs.Rig) -> None:
|
||||
sent = [_req(session="20261023-170448-5b71") for _ in range(7)]
|
||||
answers = [rs.http(rig, "POST", "/coach/requests", r) for r in sent]
|
||||
rs.wait_for(lambda: rig.relay.lane.busy is None)
|
||||
assert [a[0] for a in answers[:6]] == [202] * 6
|
||||
status, limited = answers[6]
|
||||
assert status == 200
|
||||
assert limited["code"] == "rate_limited"
|
||||
assert limited["retry"] is True
|
||||
assert limited["retry_after_ms"] == 500_000
|
||||
first = rs.http(rig, "GET", f"/coach/requests/{sent[0]['rid']}")[1]["result"]
|
||||
assert first["code"] == "not_live"
|
||||
assert rs.http(rig, "GET", f"/coach/requests/{sent[1]['rid']}")[1] == {
|
||||
"result": None
|
||||
}
|
||||
rig.relay.lane.busy = sent[1]["rid"]
|
||||
assert rs.http(rig, "GET", "/coach/heartbeat")[1]["busy"] == sent[1]["rid"]
|
||||
rig.relay.lane.busy = None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def phone(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> Iterator[tuple[rs.Rig, FakeRtdb]]:
|
||||
root = rs.fake_claude(tmp_path, monkeypatch, haiku=[], coach=[])
|
||||
with FakeRtdb.start() as fake:
|
||||
made = rs.make_rig(tmp_path, heartbeat_s=0.05, rescan_s=0.1)
|
||||
made.extra["claude"] = root
|
||||
made.relay.link_phone(Rtdb(fake.url, lambda: "tok"))
|
||||
yield made, fake
|
||||
made.relay.stop()
|
||||
|
||||
|
||||
def _result(fake: FakeRtdb, rid: str) -> dict[str, Any]:
|
||||
return dict(json.loads(rs.wait_for(lambda: fake.read(f"requests/{rid}/result"))))
|
||||
|
||||
|
||||
def test_the_phone_path_checks_ids_kinds_and_sizes(
|
||||
phone: tuple[rs.Rig, FakeRtdb],
|
||||
) -> None:
|
||||
rig, fake = phone
|
||||
fake.write("requests/not-a-ulid/req", {"kind": "hint", "dev": rs.DEV, "body": {}})
|
||||
unknown = rs.rid()
|
||||
fake.write(f"requests/{unknown}/req", {"kind": "shell", "dev": rs.DEV, "body": {}})
|
||||
assert _result(fake, unknown)["code"] == "bad_request"
|
||||
big = rs.rid()
|
||||
fake.write(
|
||||
f"requests/{big}/req",
|
||||
{"kind": "chat", "dev": rs.DEV, "body": {"text": "x" * 17000}},
|
||||
)
|
||||
assert _result(fake, big)["code"] == "bad_request"
|
||||
ended = rs.rid()
|
||||
fake.write(
|
||||
f"requests/{ended}/req",
|
||||
{"kind": "hint", "dev": rs.DEV, "body": {"session": "20261023-170448-5b71"}},
|
||||
)
|
||||
first = _result(fake, ended)
|
||||
assert first["code"] == "not_live"
|
||||
fake.write(f"requests/{ended}/result", None)
|
||||
fake.write(
|
||||
f"requests/{ended}/req",
|
||||
{"kind": "hint", "dev": rs.DEV, "body": {"session": "20261023-170448-5b71"}},
|
||||
)
|
||||
assert _result(fake, ended) == first # re-answered from the done-map, not re-run
|
||||
assert fake.read("requests/not-a-ulid/result") is None
|
||||
assert rs.calls(rig.extra["claude"]) == []
|
||||
100
leetcode_coach/leetcode_coach/tests/test_relay_main.py
Normal file
100
leetcode_coach/leetcode_coach/tests/test_relay_main.py
Normal file
@ -0,0 +1,100 @@
|
||||
"""The service entry point and the phone link's sign-in retries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import signal
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import crdt_sync
|
||||
|
||||
from leetcode_coach.relay import __main__ as entry
|
||||
from leetcode_coach.relay import app as app_mod
|
||||
from leetcode_coach.relay import config
|
||||
from leetcode_coach.relay.rtdb import Rtdb
|
||||
from leetcode_coach.tests import _relay_support as rs
|
||||
from leetcode_coach.tests._fake_rtdb import FakeRtdb
|
||||
from leetcode_guard import _snapshot_auth
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_make_rtdb_signs_in_as_its_own_app(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv(config.RTDB_ENV, "")
|
||||
seen: list[str] = []
|
||||
|
||||
class Auth:
|
||||
@staticmethod
|
||||
def id_token() -> str:
|
||||
return "live"
|
||||
|
||||
class Client:
|
||||
auth = Auth()
|
||||
|
||||
def connect(app: str, *, timeout_seconds: float) -> Client:
|
||||
seen.append(app)
|
||||
assert timeout_seconds == 15
|
||||
return Client()
|
||||
|
||||
class Config:
|
||||
database_url = "https://db.example"
|
||||
|
||||
monkeypatch.setattr(_snapshot_auth, "connect", connect)
|
||||
monkeypatch.setattr(crdt_sync.FirebaseConfig, "load", lambda _path: Config())
|
||||
rtdb = entry.make_rtdb()
|
||||
assert seen == ["leetcode_coach_relay"]
|
||||
assert rtdb.url("state").startswith("https://db.example/leetcode-guard/coach/")
|
||||
|
||||
|
||||
def test_main_serves_until_sigterm(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
rs.fake_claude(tmp_path, monkeypatch, haiku=[], coach=[])
|
||||
handlers: dict[int, Any] = {}
|
||||
monkeypatch.setattr(signal, "signal", handlers.setdefault)
|
||||
monkeypatch.setenv(config.DIR_ENV, str(tmp_path / "relay"))
|
||||
monkeypatch.setenv(config.PORT_ENV, "0")
|
||||
with FakeRtdb.start() as fake:
|
||||
monkeypatch.setenv(config.RTDB_ENV, fake.url)
|
||||
result: list[int] = []
|
||||
runner = threading.Thread(target=lambda: result.append(entry.main()))
|
||||
runner.start()
|
||||
rs.wait_for(lambda: fake.read("state"))
|
||||
assert json.loads(fake.read("state"))["session"] is None
|
||||
assert fake.tokens[-1] == entry.FAKE_ID
|
||||
rs.wait_for(lambda: signal.SIGTERM in handlers)
|
||||
handlers[signal.SIGTERM](signal.SIGTERM, None)
|
||||
runner.join(timeout=10)
|
||||
assert result == [0]
|
||||
assert signal.SIGINT in handlers
|
||||
|
||||
|
||||
def test_connect_phone_retries_until_it_links(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.setattr(app_mod, "CONNECT_BACKOFF_S", (0.0,))
|
||||
rig = rs.make_rig(tmp_path, start=False)
|
||||
with FakeRtdb.start() as fake:
|
||||
tries: list[int] = []
|
||||
|
||||
def make() -> Rtdb:
|
||||
tries.append(1)
|
||||
if len(tries) < 3:
|
||||
msg = "INVALID_LOGIN_CREDENTIALS"
|
||||
raise RuntimeError(msg)
|
||||
return Rtdb(fake.url, lambda: "tok")
|
||||
|
||||
rig.relay.connect_phone(make)
|
||||
assert len(tries) == 3
|
||||
assert rig.relay.phone is not None
|
||||
assert "phone path retries" in caplog.text
|
||||
rig.relay.stop()
|
||||
stopped = rs.make_rig(tmp_path / "b", start=False)
|
||||
stopped.relay.stop_event.set()
|
||||
stopped.relay.connect_phone(make)
|
||||
assert stopped.relay.phone is None
|
||||
stopped.relay.stop()
|
||||
186
leetcode_coach/leetcode_coach/tests/test_relay_phone.py
Normal file
186
leetcode_coach/leetcode_coach/tests/test_relay_phone.py
Normal file
@ -0,0 +1,186 @@
|
||||
"""The RTDB mirror and intake, against a recording stand-in for `Rtdb`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay import phone as phone_mod
|
||||
from leetcode_coach.relay import sse
|
||||
from leetcode_coach.relay.done import DAY_MS, DoneMap
|
||||
from leetcode_coach.relay.lane import Lane
|
||||
from leetcode_coach.relay.rtdb import RtdbError
|
||||
from leetcode_coach.tests._relay_direct import FakeAsk, make_coach, started
|
||||
from leetcode_coach.tests._relay_support import DEV, rid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Recorder:
|
||||
"""put/patch/delete/get kept in order; `fail` names calls that raise."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, str, object]] = []
|
||||
self.fail: set[str] = set()
|
||||
self.nodes: object = None
|
||||
self.events: list[sse.Event] = []
|
||||
|
||||
def _do(self, method: str, path: str, value: object = None) -> None:
|
||||
if method in self.fail or f"{method} {path}" in self.fail:
|
||||
msg = f"{method} {path}: HTTP 503"
|
||||
raise RtdbError(msg)
|
||||
self.calls.append((method, path, value))
|
||||
|
||||
def put(self, path: str, value: object) -> None:
|
||||
self._do("put", path, value)
|
||||
|
||||
def patch(self, path: str, value: dict[str, object]) -> None:
|
||||
self._do("patch", path, value)
|
||||
|
||||
def delete(self, path: str) -> None:
|
||||
self._do("delete", path)
|
||||
|
||||
def get(self, path: str) -> object:
|
||||
self._do("get", path)
|
||||
return self.nodes
|
||||
|
||||
def paths(self, method: str) -> list[str]:
|
||||
return [p for m, p, _v in self.calls if m == method]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def link(tmp_path: Path) -> tuple[phone_mod.PhoneLink, Recorder, Lane]:
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": "Hi?"}]))
|
||||
now = [10 * DAY_MS]
|
||||
lane = Lane(
|
||||
coach.handle,
|
||||
DoneMap(tmp_path / "done.jsonl", lambda: now[0]),
|
||||
clock_ms=lambda: now[0],
|
||||
)
|
||||
rec = Recorder()
|
||||
made = phone_mod.PhoneLink(
|
||||
rec, lane, coach, lambda: {"version": 1}, clock_ms=lambda: now[0]
|
||||
)
|
||||
made.attach()
|
||||
return made, rec, lane
|
||||
|
||||
|
||||
def _req(kind: str = "hint") -> dict[str, Any]:
|
||||
return {"kind": kind, "dev": DEV, "body": {"session": "20261023-170448-5b71"}}
|
||||
|
||||
|
||||
def test_events_at_every_depth_reach_the_lane(link: Any) -> None:
|
||||
made, rec, lane = link
|
||||
a, b, c, d = rid(), rid(), rid(), rid()
|
||||
made.on_event(sse.Event("put", "/", {b: {"req": _req()}, a: {"req": _req()}}))
|
||||
made.on_event(sse.Event("put", f"/{c}", {"req": _req()}))
|
||||
made.on_event(sse.Event("patch", "/", {f"{d}/req": _req()}))
|
||||
made.on_event(sse.Event("put", f"/{a}/result", "ours"))
|
||||
made.on_event(sse.Event("put", "/", None))
|
||||
assert [r.rid for r in lane._queue] == [a, b, c, d]
|
||||
made.consider(rid(), {"req": [1]}) # not an object: no rid to answer under
|
||||
made.consider("bad", {"req": _req()})
|
||||
assert len(lane._queue) == 4
|
||||
made.consider(rid(), {"req": _req()})
|
||||
limited = rid()
|
||||
made.consider(limited, {"req": _req()})
|
||||
assert "rate_limited" in str(rec.calls[-1][2])
|
||||
assert rec.calls[-1][1] == f"requests/{limited}/result"
|
||||
assert rec.paths("put").count("heartbeat") == 1
|
||||
|
||||
|
||||
def test_results_go_only_to_the_phone(link: Any) -> None:
|
||||
made, rec, _lane = link
|
||||
desktop = type("R", (), {"transport": "desktop", "rid": rid()})()
|
||||
made.on_result(desktop, {"ok": True})
|
||||
phone_req = type("R", (), {"transport": "phone", "rid": rid()})()
|
||||
made.on_result(phone_req, {"ok": True})
|
||||
puts = rec.paths("put")
|
||||
assert f"requests/{phone_req.rid}/result" in puts
|
||||
assert f"requests/{desktop.rid}/result" not in puts
|
||||
|
||||
|
||||
def test_write_failures_are_logged_and_retried(
|
||||
link: Any, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
made, rec, _lane = link
|
||||
coach = made.coach
|
||||
rec.fail = {"delete"}
|
||||
with caplog.at_level(logging.WARNING):
|
||||
started(coach)
|
||||
assert "RTDB chat reset failed" in caplog.text
|
||||
assert rec.paths("patch") == []
|
||||
rec.fail = {"patch"}
|
||||
made.on_state(coach.state(), coach.session)
|
||||
assert made._sent == set()
|
||||
rec.fail = set()
|
||||
made.on_state(coach.state(), coach.session)
|
||||
assert rec.calls[-2][0] == "patch"
|
||||
assert json.loads(str(rec.calls[0][2]))["schema"] == 1
|
||||
|
||||
|
||||
def test_rescan_answers_and_cleans_up(
|
||||
link: Any, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
made, rec, lane = link
|
||||
rec.fail = {"get"}
|
||||
made.rescan()
|
||||
assert "rescan failed" in caplog.text
|
||||
rec.fail = set()
|
||||
rec.nodes = "nope"
|
||||
made.rescan()
|
||||
old, fresh, lost, waiting = rid(), rid(), rid(), rid()
|
||||
lane.done.put(old, "phone", {"ok": True})
|
||||
lane._clock = lambda: 12 * DAY_MS
|
||||
lane.done._clock = lane._clock
|
||||
made._clock = lane._clock
|
||||
lane.done.put(fresh, "phone", {"ok": True})
|
||||
rec.nodes = {
|
||||
old: {"req": _req(), "result": "{}"},
|
||||
fresh: {"req": _req(), "result": "{}"},
|
||||
lost: {"req": _req(), "result": "{}"},
|
||||
waiting: {"req": _req()},
|
||||
"junk": 5,
|
||||
}
|
||||
made.rescan()
|
||||
assert sorted(rec.paths("delete")) == sorted(
|
||||
[f"requests/{old}", f"requests/{lost}"]
|
||||
)
|
||||
assert [r.rid for r in lane._queue] == [waiting]
|
||||
|
||||
|
||||
def test_a_cancelled_stream_leaves_rescans(
|
||||
link: Any, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
made, _rec, _lane = link
|
||||
|
||||
def cancelled(*_a: object) -> Iterator[sse.Event]:
|
||||
yield sse.Event("put", "/", None)
|
||||
msg = "rules"
|
||||
raise sse.StreamCancelledError(msg)
|
||||
|
||||
monkeypatch.setattr(phone_mod.sse, "listen", cancelled)
|
||||
made.listen(threading.Event())
|
||||
assert "rescans only now" in caplog.text
|
||||
|
||||
|
||||
def test_tend_beats_and_rescans_until_stopped(link: Any) -> None:
|
||||
made, rec, _lane = link
|
||||
stop = threading.Event()
|
||||
waits: list[float] = []
|
||||
|
||||
def wait(seconds: float) -> None:
|
||||
waits.append(seconds)
|
||||
if len(waits) == 3:
|
||||
stop.set()
|
||||
|
||||
made._beat_every, made._rescan_every = 1.0, 2.0
|
||||
made.tend(stop, wait)
|
||||
assert waits == [1.0, 1.0, 1.0]
|
||||
assert rec.paths("get") == ["requests", "requests"]
|
||||
76
leetcode_coach/leetcode_coach/tests/test_relay_phone_e2e.py
Normal file
76
leetcode_coach/leetcode_coach/tests/test_relay_phone_e2e.py
Normal file
@ -0,0 +1,76 @@
|
||||
"""TODO-coach-ui.md "Done", phone path: the scripted session through RTDB.
|
||||
|
||||
The phone PUTs ``requests/{rid}/req`` into an in-process fake RTDB (REST +
|
||||
SSE) and polls ``requests/{rid}/result``; the relay streams the requests and
|
||||
mirrors ``state``, ``chat`` and ``heartbeat``. Same flow and assertions as
|
||||
the desktop (``_relay_flow``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay.rtdb import Rtdb
|
||||
from leetcode_coach.tests import _relay_flow as flow
|
||||
from leetcode_coach.tests import _relay_support as rs
|
||||
from leetcode_coach.tests._fake_rtdb import FakeRtdb
|
||||
from leetcode_guard.tests._paths_fixture import _isolate_paths
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
__all__ = ["_isolate_paths"] # the guard's paths, redirected for every test here
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def phone(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> Iterator[tuple[rs.Rig, FakeRtdb]]:
|
||||
root = rs.fake_claude(tmp_path, monkeypatch, **flow.SCRIPT)
|
||||
rs.seed_guard()
|
||||
with FakeRtdb.start() as fake:
|
||||
made = rs.make_rig(tmp_path, heartbeat_s=0.1, rescan_s=0.2)
|
||||
made.extra["claude"] = root
|
||||
made.relay.link_phone(Rtdb(fake.url, lambda: "tok"))
|
||||
yield made, fake
|
||||
made.relay.stop()
|
||||
|
||||
|
||||
def phone_ends(fake: FakeRtdb) -> flow.Ends:
|
||||
def send(kind: str, **body: object) -> dict[str, Any]:
|
||||
rid = rs.rid()
|
||||
req = {
|
||||
"kind": kind,
|
||||
"dev": rs.DEV,
|
||||
"created": {".sv": "timestamp"},
|
||||
"body": body,
|
||||
}
|
||||
fake.write(f"requests/{rid}/req", req)
|
||||
raw = rs.wait_for(lambda: fake.read(f"requests/{rid}/result"), timeout=15)
|
||||
return dict(json.loads(raw))
|
||||
|
||||
def state() -> dict[str, Any]:
|
||||
return dict(json.loads(rs.wait_for(lambda: fake.read("state"))))
|
||||
|
||||
def chat(after: str | None) -> dict[str, Any]:
|
||||
lines = fake.read("chat") or {}
|
||||
return {k: v for k, v in lines.items() if after is None or k > after}
|
||||
|
||||
return flow.Ends(send, state, chat)
|
||||
|
||||
|
||||
def test_a_full_session_from_the_phone(phone: tuple[rs.Rig, FakeRtdb]) -> None:
|
||||
rig, fake = phone
|
||||
flow.run_flow(rig, phone_ends(fake))
|
||||
beat = rs.wait_for(lambda: fake.read("heartbeat"))
|
||||
assert isinstance(beat["beat"], int)
|
||||
assert beat["version"] == 1
|
||||
rs.wait_for(lambda: fake.read("heartbeat")["state_rev"] == rig.relay.coach.rev)
|
||||
assert json.loads(fake.read("state"))["rev"] == rig.relay.coach.rev
|
||||
assert all(
|
||||
set(v) <= {"role", "kind", "text", "at"} for v in fake.read("chat").values()
|
||||
)
|
||||
115
leetcode_coach/leetcode_coach/tests/test_relay_protocol.py
Normal file
115
leetcode_coach/leetcode_coach/tests/test_relay_protocol.py
Normal file
@ -0,0 +1,115 @@
|
||||
"""Request validation and result shapes (DOCS-coach-relay.md sections 2 and 4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay import protocol
|
||||
|
||||
RID = "01JAXY0000000000000000000Z"
|
||||
DEV = "3f2a7c1e-9b4d-4e6f-8a1b-2c3d4e5f6a7b"
|
||||
SID = "20261023-170448-5b71"
|
||||
|
||||
|
||||
def _req(kind: str, body: dict[str, Any], **extra: object) -> dict[str, Any]:
|
||||
return {"rid": RID, "kind": kind, "dev": DEV, "body": body, **extra}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rid",
|
||||
["", "short", "01JAXY000000000000000000IL", 7, None, "01jaxy0000000000000000000z"],
|
||||
)
|
||||
def test_a_bad_rid_gets_no_result(rid: object) -> None:
|
||||
with pytest.raises(protocol.BadRidError):
|
||||
protocol.parse({**_req("hint", {"session": SID}), "rid": rid}, "phone")
|
||||
with pytest.raises(protocol.BadRidError):
|
||||
protocol.parse([1], "phone")
|
||||
|
||||
|
||||
def test_a_good_request_drops_unknown_fields() -> None:
|
||||
req = protocol.parse(
|
||||
_req("chat", {"session": SID, "text": "hi", "evil": 1}, created=5, x=1),
|
||||
"desktop",
|
||||
)
|
||||
assert req == protocol.Request(
|
||||
RID, "chat", DEV, {"session": SID, "text": "hi"}, "desktop"
|
||||
)
|
||||
|
||||
|
||||
GOOD = [
|
||||
("session_start", {"slug": "missing-number"}),
|
||||
("session_end", {"session": SID}),
|
||||
("hint", {"session": SID}),
|
||||
("reveal", {"session": SID}),
|
||||
("chat", {"session": SID, "text": "x" * 8192}),
|
||||
("run_code", {"session": SID, "code": "var f = 1"}),
|
||||
("grade_step", {"session": SID, "step": 1, "answer": "n <= 10^4"}),
|
||||
("grade_step", {"session": SID, "step": 3, "picks": ["XOR"] * 30}),
|
||||
("grade_step", {"session": SID, "step": 4, "force": True}),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("kind", "body"), GOOD)
|
||||
def test_every_kind_parses(kind: str, body: dict[str, Any]) -> None:
|
||||
assert protocol.parse(_req(kind, body), "phone").body == body
|
||||
|
||||
|
||||
BAD = [
|
||||
("rm -rf", {}),
|
||||
("session_start", {"slug": "Missing Number"}),
|
||||
("session_start", {"slug": "a" * 101}),
|
||||
("session_start", {}),
|
||||
("chat", {"session": "../../etc", "text": "hi"}),
|
||||
("chat", {"session": SID, "text": "é" * 4097}),
|
||||
("chat", {"session": SID, "text": " "}),
|
||||
("chat", {"session": SID, "text": 5}),
|
||||
("grade_step", {"session": SID, "step": 4, "answer": "x"}),
|
||||
("grade_step", {"session": SID, "step": 6, "answer": "x"}),
|
||||
("grade_step", {"session": SID, "step": True, "answer": "x"}),
|
||||
("grade_step", {"session": SID, "step": 3, "picks": ["XOR"] * 31}),
|
||||
("grade_step", {"session": SID, "step": 3, "picks": ["x" * 65]}),
|
||||
("grade_step", {"session": SID, "step": 3, "picks": "XOR"}),
|
||||
("grade_step", {"session": SID, "step": 2, "force": "yes"}),
|
||||
("run_code", {"session": SID}),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("kind", "body"), BAD)
|
||||
def test_bad_bodies_are_bad_requests(kind: str, body: dict[str, Any]) -> None:
|
||||
with pytest.raises(protocol.BadRequestError):
|
||||
protocol.parse(_req(kind, body), "phone")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
_req("hint", {"session": SID}, dev="not-a-uuid"),
|
||||
{**_req("hint", {}), "body": []},
|
||||
{**_req("hint", {"session": SID}), "padding": "x" * 17000},
|
||||
],
|
||||
)
|
||||
def test_dev_body_and_size_are_checked(raw: dict[str, Any]) -> None:
|
||||
with pytest.raises(protocol.BadRequestError):
|
||||
protocol.parse(raw, "phone")
|
||||
|
||||
|
||||
def test_results_carry_time_and_rev() -> None:
|
||||
assert protocol.ok({"a": 1}, at=5, state_rev=2) == {
|
||||
"ok": True,
|
||||
"at": 5,
|
||||
"state_rev": 2,
|
||||
"data": {"a": 1},
|
||||
}
|
||||
err = protocol.failure("rate_limited", "slow down", at=7, retry=True, after=3)
|
||||
assert err == {
|
||||
"ok": False,
|
||||
"at": 7,
|
||||
"code": "rate_limited",
|
||||
"error": "slow down",
|
||||
"retry": True,
|
||||
"after": 3,
|
||||
}
|
||||
assert protocol.ok_retryable("model_error")
|
||||
assert not protocol.ok_retryable("bad_request")
|
||||
209
leetcode_coach/leetcode_coach/tests/test_relay_rtdb.py
Normal file
209
leetcode_coach/leetcode_coach/tests/test_relay_rtdb.py
Normal file
@ -0,0 +1,209 @@
|
||||
"""The relay's RTDB REST client, against the in-process fake database."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from leetcode_coach.relay import rtdb
|
||||
from leetcode_coach.tests._fake_rtdb import NOW_MS, FakeRtdb
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
SECRET = "tok-secret"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake() -> Iterator[FakeRtdb]:
|
||||
with FakeRtdb.start() as db:
|
||||
yield db
|
||||
|
||||
|
||||
def client(fake: FakeRtdb, **kwargs: Any) -> rtdb.Rtdb:
|
||||
counter = itertools.count(1)
|
||||
return rtdb.Rtdb(fake.url, lambda: f"{SECRET}-{next(counter)}", **kwargs)
|
||||
|
||||
|
||||
def test_url_quotes_each_segment_under_the_coach_root() -> None:
|
||||
db = rtdb.Rtdb("https://x.firebaseio.com/", lambda: "t")
|
||||
assert db.url("requests/01AB/result") == (
|
||||
"https://x.firebaseio.com/leetcode-guard/coach/requests/01AB/result.json"
|
||||
)
|
||||
assert (
|
||||
db.url("/a b//c/")
|
||||
== "https://x.firebaseio.com/leetcode-guard/coach/a%20b/c.json"
|
||||
)
|
||||
assert db.url("") == "https://x.firebaseio.com/leetcode-guard/coach.json"
|
||||
|
||||
|
||||
def test_put_and_get_a_json_string_leaf(fake: FakeRtdb) -> None:
|
||||
db = client(fake)
|
||||
db.put("state", json.dumps({"schema": 1, "steps": []}))
|
||||
assert fake.read("state") == '{"schema": 1, "steps": []}'
|
||||
assert db.get("state") == '{"schema": 1, "steps": []}'
|
||||
method, path, query = fake.requests[0]
|
||||
assert (method, path) == ("PUT", "/leetcode-guard/coach/state.json")
|
||||
assert "print=silent" in query
|
||||
|
||||
|
||||
def test_get_of_a_missing_node_is_none(fake: FakeRtdb) -> None:
|
||||
assert client(fake).get("requests") is None
|
||||
|
||||
|
||||
def test_server_timestamps_resolve(fake: FakeRtdb) -> None:
|
||||
client(fake).put("heartbeat", {"beat": rtdb.SERVER_TIME, "version": 1})
|
||||
assert fake.read("heartbeat") == {"beat": NOW_MS, "version": 1}
|
||||
|
||||
|
||||
def test_patch_merges_children_and_nested_keys(fake: FakeRtdb) -> None:
|
||||
db = client(fake)
|
||||
db.patch("chat", {"m0001": {"role": "coach", "text": "hi"}})
|
||||
db.patch("chat", {"m0002": {"role": "kuhy", "text": "yo"}, "m0001/text": "hey"})
|
||||
assert fake.read("chat") == {
|
||||
"m0001": {"role": "coach", "text": "hey"},
|
||||
"m0002": {"role": "kuhy", "text": "yo"},
|
||||
}
|
||||
assert "print=silent" in fake.requests[-1][2]
|
||||
|
||||
|
||||
def test_delete_removes_the_node_and_prunes_empty_parents(fake: FakeRtdb) -> None:
|
||||
db = client(fake)
|
||||
db.put("requests/R1/result", '"done"')
|
||||
db.delete("requests/R1")
|
||||
assert fake.read("requests") is None
|
||||
assert fake.read() is None
|
||||
|
||||
|
||||
def test_every_call_asks_for_a_fresh_token(fake: FakeRtdb) -> None:
|
||||
db = client(fake)
|
||||
db.get("a")
|
||||
db.get("b")
|
||||
assert fake.tokens == [f"{SECRET}-1", f"{SECRET}-2"]
|
||||
|
||||
|
||||
def test_an_http_error_names_method_path_and_status_only(fake: FakeRtdb) -> None:
|
||||
fake.fail_next(1, status=401)
|
||||
with pytest.raises(rtdb.RtdbError) as caught:
|
||||
client(fake).put("state", "x")
|
||||
assert str(caught.value) == "PUT state: HTTP 401"
|
||||
|
||||
|
||||
def test_a_network_error_never_leaks_the_url_or_token() -> None:
|
||||
db = rtdb.Rtdb("http://127.0.0.1:9", lambda: SECRET, timeout_s=2.0)
|
||||
with pytest.raises(rtdb.RtdbError) as caught:
|
||||
db.get("requests")
|
||||
assert str(caught.value) == "GET requests: ConnectionError"
|
||||
assert caught.value.__cause__ is None
|
||||
assert caught.value.__suppress_context__
|
||||
|
||||
|
||||
class _CannedSession(requests.Session):
|
||||
"""Answers every request with one canned response (or raises)."""
|
||||
|
||||
def __init__(self, response: requests.Response) -> None:
|
||||
super().__init__()
|
||||
self.response = response
|
||||
|
||||
def request(self, *args: Any, **kwargs: Any) -> requests.Response:
|
||||
return self.response
|
||||
|
||||
def get(self, *args: Any, **kwargs: Any) -> requests.Response:
|
||||
return self.response
|
||||
|
||||
|
||||
def _response(status: int, body: bytes) -> requests.Response:
|
||||
response = requests.Response()
|
||||
response.status_code = status
|
||||
response._content = body
|
||||
return response
|
||||
|
||||
|
||||
def test_a_get_whose_body_is_not_json_is_an_error() -> None:
|
||||
db = rtdb.Rtdb(
|
||||
"http://x", lambda: "t", session=_CannedSession(_response(200, b"<html>"))
|
||||
)
|
||||
with pytest.raises(rtdb.RtdbError, match=r"^GET state: not JSON$"):
|
||||
db.get("state")
|
||||
|
||||
|
||||
def test_stream_starts_with_the_whole_node_then_reports_writes(fake: FakeRtdb) -> None:
|
||||
fake.write("requests/R0/req", {"kind": "chat"})
|
||||
lines = client(fake).stream("requests")
|
||||
kind, data = next(lines)
|
||||
assert kind == "put"
|
||||
assert json.loads(data) == {"path": "/", "data": {"R0": {"req": {"kind": "chat"}}}}
|
||||
fake.write("requests/R1/req", {"kind": "hint", "created": {".sv": "timestamp"}})
|
||||
kind, data = next(lines)
|
||||
assert (kind, json.loads(data)) == (
|
||||
"put",
|
||||
{"path": "/R1/req", "data": {"kind": "hint", "created": NOW_MS}},
|
||||
)
|
||||
fake.revoke_streams()
|
||||
assert next(lines)[0] == "auth_revoked"
|
||||
assert list(lines) == []
|
||||
|
||||
|
||||
def test_stream_http_error(fake: FakeRtdb) -> None:
|
||||
fake.fail_next(1, status=503)
|
||||
with pytest.raises(rtdb.RtdbError, match=r"^STREAM requests: HTTP 503$"):
|
||||
next(client(fake).stream("requests"))
|
||||
|
||||
|
||||
def test_stream_connect_error() -> None:
|
||||
db = rtdb.Rtdb("http://127.0.0.1:9", lambda: SECRET)
|
||||
with pytest.raises(rtdb.RtdbError, match=r"^STREAM requests: ConnectionError$"):
|
||||
next(db.stream("requests"))
|
||||
|
||||
|
||||
class _DyingResponse(requests.Response):
|
||||
"""Delivers one block, then the connection drops mid-read."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.status_code = 200
|
||||
self.closed_by_client = False
|
||||
|
||||
def iter_lines(self, *args: Any, **kwargs: Any) -> Iterator[bytes]:
|
||||
yield b"" # a stray blank line before any field is no block
|
||||
yield b"event: keep-alive"
|
||||
yield b"data: null"
|
||||
yield b""
|
||||
raise requests.exceptions.ChunkedEncodingError
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed_by_client = True
|
||||
|
||||
|
||||
def test_a_drop_mid_stream_is_an_rtdb_error_and_closes_the_response() -> None:
|
||||
response = _DyingResponse()
|
||||
db = rtdb.Rtdb("http://x", lambda: "t", session=_CannedSession(response))
|
||||
lines = db.stream("requests")
|
||||
assert next(lines) == ("keep-alive", "null")
|
||||
with pytest.raises(
|
||||
rtdb.RtdbError, match=r"^STREAM requests: ChunkedEncodingError$"
|
||||
):
|
||||
next(lines)
|
||||
assert response.closed_by_client
|
||||
|
||||
|
||||
def test_stream_reports_patches_deletes_and_writes_above_the_node(
|
||||
fake: FakeRtdb,
|
||||
) -> None:
|
||||
db = client(fake)
|
||||
lines = db.stream("requests")
|
||||
assert json.loads(next(lines)[1])["data"] is None
|
||||
db.patch("requests", {"R1/req": {"kind": "chat"}})
|
||||
assert json.loads(next(lines)[1]) == {
|
||||
"path": "/",
|
||||
"data": {"R1/req": {"kind": "chat"}},
|
||||
}
|
||||
db.delete("requests/R1")
|
||||
assert json.loads(next(lines)[1]) == {"path": "/R1", "data": None}
|
||||
db.put("", {"requests": {"R2": {"req": 1}}})
|
||||
assert json.loads(next(lines)[1]) == {"path": "/", "data": {"R2": {"req": 1}}}
|
||||
lines.close()
|
||||
202
leetcode_coach/leetcode_coach/tests/test_relay_runcode.py
Normal file
202
leetcode_coach/leetcode_coach/tests/test_relay_runcode.py
Normal file
@ -0,0 +1,202 @@
|
||||
"""run_code: kuhy's JavaScript against the statement's examples, under node.
|
||||
|
||||
The real node runs the behaviour tests (the permission model is the point and
|
||||
a fake cannot prove it); a fake runner covers the failure branches.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay import runcode
|
||||
from leetcode_coach.tests._relay_statements import MISSING_NUMBER, TWO_SUM
|
||||
|
||||
NODE = shutil.which("node") or "/usr/bin/node"
|
||||
needs_node = pytest.mark.skipif(not shutil.which("node"), reason="node not on PATH")
|
||||
|
||||
RIGHT = """var missingNumber = function(nums) {
|
||||
let x = nums.length;
|
||||
for (let i = 0; i < nums.length; i++) x ^= i ^ nums[i];
|
||||
return x;
|
||||
};"""
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_a_correct_solution_passes_every_example() -> None:
|
||||
out = runcode.run(RIGHT, MISSING_NUMBER, node=NODE)
|
||||
assert out == {
|
||||
"examples": [
|
||||
{"n": 1, "ok": True, "got": "2", "expected": "2"},
|
||||
{"n": 2, "ok": True, "got": "2", "expected": "2"},
|
||||
{"n": 3, "ok": True, "got": "8", "expected": "8"},
|
||||
],
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_a_wrong_answer_is_got_versus_expected() -> None:
|
||||
out = runcode.run("var twoSum = (nums, t) => [1, 0];", TWO_SUM, node=NODE)
|
||||
first = out["examples"][0]
|
||||
assert first == {"n": 1, "ok": False, "got": "[1,0]", "expected": "[0,1]"}
|
||||
assert out["error"] is None
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_console_output_does_not_corrupt_results() -> None:
|
||||
code = (
|
||||
"var missingNumber = function(nums) {\n"
|
||||
" console.log('2'); console.error('noise'); process.stdout.write('8\\n');\n"
|
||||
" return 0;\n};"
|
||||
)
|
||||
out = runcode.run(code, MISSING_NUMBER, node=NODE)
|
||||
assert [e["got"] for e in out["examples"]] == ["0", "0", "0"]
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_undefined_result_is_shown_as_undefined() -> None:
|
||||
out = runcode.run(
|
||||
"var missingNumber = function(nums) {};", MISSING_NUMBER, node=NODE
|
||||
)
|
||||
assert out["examples"][0]["got"] == "undefined"
|
||||
assert out["examples"][0]["ok"] is False
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_a_syntax_error_reports_syntax_and_no_examples() -> None:
|
||||
out = runcode.run("var missingNumber = function(nums) {", MISSING_NUMBER, node=NODE)
|
||||
assert out == {"examples": [], "error": "syntax"}
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_a_thrown_error_shows_only_its_class_name() -> None:
|
||||
code = (
|
||||
"var missingNumber = function(nums) { throw new TypeError('secret detail'); };"
|
||||
)
|
||||
out = runcode.run(code, MISSING_NUMBER, node=NODE)
|
||||
assert out["examples"][0]["got"] == "error: TypeError"
|
||||
assert "secret" not in str(out)
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_a_top_level_crash_marks_every_example() -> None:
|
||||
code = "null.x;\nvar missingNumber = function(nums) { return 2; };"
|
||||
out = runcode.run(code, MISSING_NUMBER, node=NODE)
|
||||
assert [e["got"] for e in out["examples"]] == ["error: TypeError"] * 3
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_reading_outside_its_directory_is_denied() -> None:
|
||||
code = (
|
||||
"var missingNumber = function(nums) {\n"
|
||||
" return process.mainModule.require('fs').readFileSync('/etc/passwd', 'utf8');\n};"
|
||||
)
|
||||
out = runcode.run(code, MISSING_NUMBER, node=NODE)
|
||||
assert out["examples"][0]["got"] == "error: Error"
|
||||
assert "root" not in str(out)
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_a_missing_function_is_an_error_per_example() -> None:
|
||||
out = runcode.run("var other = () => 1;", MISSING_NUMBER, node=NODE)
|
||||
assert out["examples"][0]["got"] == "error: ReferenceError"
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_an_infinite_loop_times_out() -> None:
|
||||
code = "var missingNumber = function(nums) { for (;;) {} };"
|
||||
out = runcode.run(code, MISSING_NUMBER, node=NODE, timeout_s=1.0)
|
||||
assert out == {"examples": [], "error": "timeout"}
|
||||
|
||||
|
||||
@needs_node
|
||||
def test_a_long_result_is_truncated() -> None:
|
||||
code = "var missingNumber = function(nums) { return 'x'.repeat(5000); };"
|
||||
got = runcode.run(code, MISSING_NUMBER, node=NODE)["examples"][0]["got"]
|
||||
assert len(got) == runcode.MAX_SHOWN + 3
|
||||
assert got.endswith("...")
|
||||
|
||||
|
||||
def test_no_examples_is_reported_without_running() -> None:
|
||||
def never(*_a: Any, **_k: Any) -> subprocess.CompletedProcess[bytes]:
|
||||
raise AssertionError
|
||||
|
||||
assert runcode.run("x", "no examples here", runner=never) == {
|
||||
"examples": [],
|
||||
"error": "no_examples",
|
||||
}
|
||||
|
||||
|
||||
def test_no_function_name_is_no_examples() -> None:
|
||||
text = "**Input:** n = 1\n**Output:** 1\n"
|
||||
assert runcode.run("x", text)["error"] == "no_examples"
|
||||
|
||||
|
||||
def test_node_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
||||
monkeypatch.setenv("LEETCODE_NODE_BIN", str(tmp_path / "no-node"))
|
||||
assert runcode.run(RIGHT, MISSING_NUMBER) == {
|
||||
"examples": [],
|
||||
"error": "node_missing",
|
||||
}
|
||||
|
||||
|
||||
def test_node_from_path_then_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("LEETCODE_NODE_BIN", raising=False)
|
||||
monkeypatch.setattr(shutil, "which", lambda _name: None)
|
||||
assert runcode.node_bin() == "/usr/bin/node"
|
||||
monkeypatch.setattr(shutil, "which", lambda _name: "/opt/node")
|
||||
assert runcode.node_bin() == "/opt/node"
|
||||
|
||||
|
||||
def test_the_argv_is_fixed_and_the_env_minimal() -> None:
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
def fake(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[bytes]:
|
||||
seen.update(argv=argv, **kwargs)
|
||||
return subprocess.CompletedProcess(argv, 0, b"", b"")
|
||||
|
||||
runcode.run(RIGHT, MISSING_NUMBER, node="/n", runner=fake)
|
||||
tmp = seen["cwd"]
|
||||
assert seen["argv"] == [
|
||||
"/n",
|
||||
"--permission",
|
||||
f"--allow-fs-read={tmp}",
|
||||
"--max-old-space-size=128",
|
||||
f"{tmp}/run.js",
|
||||
]
|
||||
assert seen["env"] == {"PATH": "/usr/bin:/bin"}
|
||||
assert seen["stdin"] is subprocess.DEVNULL
|
||||
assert "shell" not in seen
|
||||
|
||||
|
||||
def _marked(stdout_for_token: Any) -> Any:
|
||||
def fake(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[bytes]:
|
||||
script = kwargs["cwd"] + "/run.js"
|
||||
with open(script, encoding="utf-8") as fh:
|
||||
token = fh.read().split('const __mark = "', 1)[1].split('"', 1)[0]
|
||||
return subprocess.CompletedProcess(
|
||||
argv, 0, stdout_for_token(token).encode(), b""
|
||||
)
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
def test_output_past_the_cap_is_dropped() -> None:
|
||||
def big(token: str) -> str:
|
||||
return f"{token} 1 ok 2\n" + "y" * runcode.MAX_OUTPUT + f"\n{token} 2 ok 2\n"
|
||||
|
||||
out = runcode.run(RIGHT, MISSING_NUMBER, runner=_marked(big))
|
||||
assert [e["got"] for e in out["examples"]] == ["2", "no result", "no result"]
|
||||
|
||||
|
||||
def test_forged_and_malformed_lines_are_ignored() -> None:
|
||||
def odd(token: str) -> str:
|
||||
return f"fake 1 ok 9\n{token} x ok 1\n{token} 1 ok {{bad\n{token} 2\n"
|
||||
|
||||
out = runcode.run(RIGHT, MISSING_NUMBER, runner=_marked(odd))
|
||||
assert [e["got"] for e in out["examples"]] == ["{bad", "no result", "no result"]
|
||||
assert out["examples"][0]["ok"] is False
|
||||
169
leetcode_coach/leetcode_coach/tests/test_relay_sse.py
Normal file
169
leetcode_coach/leetcode_coach/tests/test_relay_sse.py
Normal file
@ -0,0 +1,169 @@
|
||||
"""SSE parsing and the reconnect-forever listener."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach.relay import rtdb, sse
|
||||
from leetcode_coach.tests._fake_rtdb import FakeRtdb
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
Script = list[tuple[str, str] | Exception] | Exception
|
||||
|
||||
|
||||
class ScriptedRtdb:
|
||||
"""Each `stream` call plays the next connection: blocks, then EOF or a raise."""
|
||||
|
||||
def __init__(self, *connections: Script, on_last: threading.Event) -> None:
|
||||
self.connections = list(connections)
|
||||
self.on_last = on_last
|
||||
self.opened = 0
|
||||
|
||||
def stream(self, path: str) -> Generator[tuple[str, str]]:
|
||||
assert path == "requests"
|
||||
self.opened += 1
|
||||
script = self.connections.pop(0)
|
||||
if not self.connections:
|
||||
self.on_last.set() # the listener stops after this connection
|
||||
if isinstance(script, Exception):
|
||||
raise script
|
||||
for item in script:
|
||||
if isinstance(item, Exception):
|
||||
raise item
|
||||
yield item
|
||||
|
||||
|
||||
def put(path: str, data: object) -> tuple[str, str]:
|
||||
return "put", f'{{"path": "{path}", "data": {data}}}'
|
||||
|
||||
|
||||
def run(*connections: Script) -> tuple[list[sse.Event], list[float], ScriptedRtdb]:
|
||||
stop = threading.Event()
|
||||
db = ScriptedRtdb(*connections, on_last=stop)
|
||||
sleeps: list[float] = []
|
||||
events = list(sse.listen(db, "requests", stop, sleeps.append))
|
||||
return events, sleeps, db
|
||||
|
||||
|
||||
def test_parse_joins_data_lines_and_defaults_to_message() -> None:
|
||||
assert sse.parse(["event: put", 'data: {"a":', "data: 1}"]) == ("put", '{"a":\n1}')
|
||||
assert sse.parse([": comment", "id: 7", "data:x"]) == ("message", "x")
|
||||
|
||||
|
||||
def test_put_and_patch_events_come_out_typed_and_noise_is_ignored() -> None:
|
||||
events, sleeps, _db = run(
|
||||
[
|
||||
("keep-alive", "null"),
|
||||
put("/", "null"),
|
||||
("patch", '{"path": "/R1", "data": {"req": 1}}'),
|
||||
("rules_debug", "x"),
|
||||
],
|
||||
[put("/", "{}")],
|
||||
)
|
||||
assert events == [
|
||||
sse.Event("put", "/", None),
|
||||
sse.Event("patch", "/R1", {"req": 1}),
|
||||
sse.Event("put", "/", {}),
|
||||
]
|
||||
assert sleeps == [] # a connection that delivered events reconnects at once
|
||||
|
||||
|
||||
def test_a_payload_without_a_path_is_at_the_root() -> None:
|
||||
events, _sleeps, _db = run([("put", '{"data": 5}')])
|
||||
assert events == [sse.Event("put", "/", 5)]
|
||||
|
||||
|
||||
def test_auth_revoked_reconnects_at_once() -> None:
|
||||
events, sleeps, db = run([("auth_revoked", "null")], [put("/R2", '"x"')])
|
||||
assert events == [sse.Event("put", "/R2", "x")]
|
||||
assert (sleeps, db.opened) == ([], 2)
|
||||
|
||||
|
||||
def test_cancel_is_fatal() -> None:
|
||||
stop = threading.Event()
|
||||
db = ScriptedRtdb([("cancel", "null")], [], on_last=threading.Event())
|
||||
with pytest.raises(sse.StreamCancelledError, match="cancelled"):
|
||||
list(sse.listen(db, "requests", stop, lambda _s: None))
|
||||
assert isinstance(sse.StreamCancelledError("x"), rtdb.RtdbError)
|
||||
|
||||
|
||||
def test_failures_back_off_and_an_event_resets_the_count() -> None:
|
||||
boom = rtdb.RtdbError("STREAM requests: HTTP 503")
|
||||
events, sleeps, _db = run(
|
||||
boom,
|
||||
boom,
|
||||
[("put", "not json")],
|
||||
[("put", "[1, 2]")],
|
||||
[put("/", "1")],
|
||||
boom,
|
||||
[],
|
||||
*[boom] * 4,
|
||||
[],
|
||||
)
|
||||
assert events == [sse.Event("put", "/", 1)]
|
||||
assert sleeps == [1.0, 2.0, 5.0, 10.0, 1.0, 2.0, 5.0, 10.0, 30.0, 30.0]
|
||||
|
||||
|
||||
def test_a_clean_close_with_no_event_backs_off() -> None:
|
||||
_events, sleeps, _db = run([], [], [])
|
||||
assert sleeps == [1.0, 2.0] # the last close sees `stop` and never sleeps
|
||||
|
||||
|
||||
def test_an_error_after_events_retries_from_the_shortest_delay() -> None:
|
||||
boom = rtdb.RtdbError("STREAM requests: ReadTimeout")
|
||||
events, sleeps, _db = run([put("/", "1"), boom], boom, [])
|
||||
assert events == [sse.Event("put", "/", 1)]
|
||||
assert sleeps == [1.0, 2.0]
|
||||
|
||||
|
||||
def test_stop_ends_the_listener_after_the_current_event() -> None:
|
||||
stop = threading.Event()
|
||||
db = ScriptedRtdb([put("/", "1"), put("/", "2")], [], on_last=threading.Event())
|
||||
listener = sse.listen(db, "requests", stop, lambda _s: None)
|
||||
assert next(listener) == sse.Event("put", "/", 1)
|
||||
stop.set()
|
||||
assert list(listener) == []
|
||||
assert db.opened == 1
|
||||
|
||||
|
||||
def test_an_already_stopped_listener_opens_nothing() -> None:
|
||||
stop = threading.Event()
|
||||
stop.set()
|
||||
db = ScriptedRtdb([], on_last=threading.Event())
|
||||
assert list(sse.listen(db, "requests", stop)) == []
|
||||
assert db.opened == 0
|
||||
|
||||
|
||||
def test_the_default_sleep_waits_on_the_stop_event() -> None:
|
||||
stop = threading.Event()
|
||||
db = ScriptedRtdb(rtdb.RtdbError("down"), on_last=stop)
|
||||
assert (
|
||||
list(sse.listen(db, "requests", stop)) == []
|
||||
) # stop.wait(1.0) returns at once
|
||||
|
||||
|
||||
def test_against_the_fake_database_across_a_token_revocation() -> None:
|
||||
counter = itertools.count(1)
|
||||
with FakeRtdb.start() as fake:
|
||||
db = rtdb.Rtdb(fake.url, lambda: f"t{next(counter)}")
|
||||
stop = threading.Event()
|
||||
listener = sse.listen(db, "requests", stop, lambda _s: None)
|
||||
assert next(listener) == sse.Event("put", "/", None)
|
||||
fake.write("requests/R1/req", {"kind": "chat"})
|
||||
assert next(listener) == sse.Event("put", "/R1/req", {"kind": "chat"})
|
||||
fake.revoke_streams()
|
||||
assert next(listener) == sse.Event(
|
||||
"put", "/", {"R1": {"req": {"kind": "chat"}}}
|
||||
)
|
||||
assert fake.tokens[-1] == "t2" # the reconnect asked for a fresh token
|
||||
fake.write("requests/R1/result", "ok")
|
||||
assert next(listener) == sse.Event("put", "/R1/result", "ok")
|
||||
stop.set()
|
||||
fake.write("requests/R2/req", {"kind": "hint"})
|
||||
assert list(listener) == []
|
||||
171
leetcode_coach/leetcode_coach/tests/test_relay_units.py
Normal file
171
leetcode_coach/leetcode_coach/tests/test_relay_units.py
Normal file
@ -0,0 +1,171 @@
|
||||
"""Toolbox grading on the real key, the Accepted probe, and smaller seams."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from leetcode_coach import answer_key, leak, toolbox
|
||||
from leetcode_coach.relay import chat, config, probe, toolbox_grade, turns
|
||||
from leetcode_coach.tests._relay_direct import FakeAsk, make_coach, req, started
|
||||
from leetcode_coach.tests._relay_support import Clock, FakeProbe
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
REAL = toolbox.load_safe(toolbox.TOOLBOX_PATH)
|
||||
KEYS = answer_key.load_answers()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", sorted(KEYS))
|
||||
def test_every_key_passes_its_own_entries_and_names_none(slug: str) -> None:
|
||||
key = KEYS[slug]
|
||||
assert toolbox_grade.grade(list(key.entries), key, REAL) == ("pass", "")
|
||||
for alt in key.also_accepted:
|
||||
assert toolbox_grade.grade([alt], key, REAL)[0] in {"pass", "partial"}
|
||||
right = {*key.entries, *key.also_accepted}
|
||||
wrong = [e.name for e in REAL.entries if e.name not in right]
|
||||
verdict, feedback = toolbox_grade.grade(wrong[:1], key, REAL)
|
||||
assert verdict in {"fail", "partial"}
|
||||
assert leak.leaked_names(feedback, [], REAL) == []
|
||||
|
||||
|
||||
def test_combined_entries_count_from_their_parts() -> None:
|
||||
combined = toolbox_grade.resolve("prefix sum + hash map", REAL)
|
||||
assert combined is not None
|
||||
parts = toolbox_grade.parts(combined, REAL)
|
||||
assert parts is not None
|
||||
assert [p.name for p in parts] == ["prefix sums", "hash map"]
|
||||
key = KEYS["subarray-sum-equals-k"]
|
||||
assert toolbox_grade.grade(["prefix sums", "hash map"], key, REAL) == ("pass", "")
|
||||
merge = toolbox_grade.resolve("sort by start + merge", REAL)
|
||||
assert merge is not None
|
||||
assert toolbox_grade.parts(merge, REAL) is None
|
||||
xor = toolbox_grade.resolve("xor", REAL)
|
||||
assert xor is not None
|
||||
assert toolbox_grade.parts(xor, REAL) is None
|
||||
|
||||
|
||||
def test_alternatives_extras_and_misses() -> None:
|
||||
key = KEYS["missing-number"]
|
||||
assert toolbox_grade.grade(["sum / difference"], key, REAL) == ("pass", "")
|
||||
assert toolbox_grade.grade(["XOR", "heap"], key, REAL) == ("pass", "")
|
||||
verdict, feedback = toolbox_grade.grade(["XOR", "heap", "gcd"], key, REAL)
|
||||
assert verdict == "partial"
|
||||
assert feedback == toolbox_grade.TOO_MANY
|
||||
verdict, feedback = toolbox_grade.grade(["heap"], key, REAL)
|
||||
assert verdict == "fail"
|
||||
assert "self-inverse" in feedback
|
||||
spread = toolbox_grade.grade(["XOR"], KEYS["single-number-iii"], REAL)
|
||||
assert spread[0] == "partial"
|
||||
box = toolbox.Toolbox(
|
||||
(toolbox.Entry("A", (), ("p",)), toolbox.Entry("B", (), ("p",))), {"p": "m"}
|
||||
)
|
||||
ghost = answer_key.Answer("r", ("B", "Ghost"), (), ("p",), ("x",))
|
||||
assert toolbox_grade.grade(["A"], ghost, box) == ("fail", toolbox_grade.SPREAD)
|
||||
|
||||
|
||||
def test_the_probe_is_three_valued_and_spaced(caplog: pytest.LogCaptureFixture) -> None:
|
||||
fake, clock = FakeProbe(), Clock()
|
||||
accepted = probe.AcceptedProbe(fake, monotonic=clock)
|
||||
assert accepted.due()
|
||||
assert accepted.check("missing-number", 1000) == probe.NOT_YET
|
||||
assert not accepted.due()
|
||||
clock.now += 60
|
||||
assert accepted.due()
|
||||
fake.status, fake.timestamp_s = "accepted", 0
|
||||
assert accepted.check("missing-number", 1000) == probe.NOT_YET # older than start
|
||||
fake.timestamp_s = 2
|
||||
assert accepted.check("missing-number", 1000) == probe.ACCEPTED
|
||||
assert accepted.check("two-sum", 1000) == probe.NOT_YET
|
||||
fake.status = "unusable"
|
||||
assert accepted.check("missing-number", 1000) == probe.UNKNOWN
|
||||
|
||||
def boom() -> object:
|
||||
msg = "dns"
|
||||
raise OSError(msg)
|
||||
|
||||
assert probe.AcceptedProbe(boom).check("x", 0) == probe.UNKNOWN
|
||||
assert "probe failed" in caplog.text
|
||||
|
||||
|
||||
def test_the_live_probe_uses_the_harvesters_client(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class Client:
|
||||
post = staticmethod(lambda *_a: None)
|
||||
username = "kuhy"
|
||||
|
||||
seen: list[str] = []
|
||||
monkeypatch.setattr(probe, "build_client", Client)
|
||||
monkeypatch.setattr(probe, "fetch_recent_ac", lambda _post, user: seen.append(user))
|
||||
probe.live_probe()
|
||||
assert seen == ["kuhy"]
|
||||
|
||||
|
||||
def test_ticks_that_must_do_nothing(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": "Hi?"}]))
|
||||
coach.tick() # no session
|
||||
sid = started(coach)
|
||||
coach.tick() # step 1 open, not 4
|
||||
session = coach.session
|
||||
assert session is not None
|
||||
for step in (1, 2, 3):
|
||||
session.log_event("grade", step=step, grade="pass", forced=False, via="haiku")
|
||||
|
||||
def vanish(*_a: object) -> str:
|
||||
coach.session = None
|
||||
return "not_yet"
|
||||
|
||||
monkeypatch.setattr(coach.deps.probe, "check", vanish)
|
||||
coach.tick() # the session went away mid-probe
|
||||
assert session.probe is None
|
||||
coach.session = session
|
||||
monkeypatch.setattr(coach.deps.probe, "due", lambda: False)
|
||||
coach.tick()
|
||||
coach.handle(req("session_end", session=sid))
|
||||
coach.tick()
|
||||
assert session.probe is None
|
||||
|
||||
|
||||
def test_a_restored_ended_session_and_a_refetched_statement(tmp_path: Path) -> None:
|
||||
fetched: list[str] = []
|
||||
|
||||
def fetch(slug: str) -> tuple[str, str]:
|
||||
fetched.append(slug)
|
||||
return "Missing Number", "statement"
|
||||
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": "Hi?"}]), fetch=fetch)
|
||||
sid = started(coach)
|
||||
coach.session.log_event("rung", rung="H1")
|
||||
coach.handle(req("session_end", session=sid))
|
||||
again = make_coach(tmp_path, FakeAsk([]), fetch=fetch)
|
||||
assert again.session is not None
|
||||
assert again.session.end_reason == "ended"
|
||||
assert again.session.statement is None
|
||||
assert turns.statement(again, again.session) == "statement"
|
||||
assert again.session.gate.known == "statement"
|
||||
assert fetched == ["missing-number", "missing-number"]
|
||||
|
||||
|
||||
def test_env_and_config_seams(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
coach = make_coach(tmp_path, FakeAsk([{"result": "Hi?"}]))
|
||||
started(coach)
|
||||
session = coach.session
|
||||
session.cluster = None
|
||||
assert chat._env(session) == {"LEETCODE_COACH_LOG": str(session.log)}
|
||||
monkeypatch.setenv(config.DIR_ENV, str(tmp_path / "d"))
|
||||
monkeypatch.setenv(config.PORT_ENV, "9999")
|
||||
monkeypatch.setenv(config.RTDB_ENV, "http://127.0.0.1:1")
|
||||
monkeypatch.setenv(config.CARDS_DRY_RUN_ENV, "1")
|
||||
assert config.state_dir() == tmp_path / "d"
|
||||
assert config.port() == 9999
|
||||
assert config.rtdb_url() == "http://127.0.0.1:1"
|
||||
assert config.cards_dry_run()
|
||||
monkeypatch.delenv(config.DIR_ENV)
|
||||
monkeypatch.delenv(config.PORT_ENV)
|
||||
assert config.state_dir() == config.DEFAULT_DIR
|
||||
assert config.port() == config.DEFAULT_PORT
|
||||
@ -33,7 +33,7 @@ def _haiku(answer: object) -> _Haiku:
|
||||
def _ollama(monkeypatch: pytest.MonkeyPatch, text: str | Exception) -> list[Any]:
|
||||
sent: list[Any] = []
|
||||
|
||||
def chat(messages: list[dict[str, str]]) -> local_backend.LocalReply:
|
||||
def chat(messages: list[dict[str, str]], **_kw: object) -> local_backend.LocalReply:
|
||||
sent.append(messages)
|
||||
if isinstance(text, Exception):
|
||||
raise text
|
||||
|
||||
Loading…
Reference in New Issue
Block a user