diff --git a/leetcode_coach/README.md b/leetcode_coach/README.md index 12f6653..8d04310 100644 --- a/leetcode_coach/README.md +++ b/leetcode_coach/README.md @@ -50,6 +50,10 @@ means paste it), `fetch_problem.render(q, lang) -> str` (markdown with - `gate.screen(block, *, is_code=None) -> (sent | None, note)`: `None` means refuse and show `note`; otherwise forward `sent` to the coach and show `note` if non-empty. Logs `req`, `req_check`, `code`, `reveal`. +- `ask` (gate, `req_check.check`, `looks_like_code`, graders) is CLI-style `(args, prompt, **kw)` + (default: haiku, then Ollama) or a structured `(system, prompt, schema) -> dict` + model function such as the tutor's `ask_model` (answers are labelled `via: "model"`; + any raise falls through to Ollama, then fails open to `unverified`). - `gate.refusal_note()`, `gate.code_seen()`, `gate.events()`, `gate.sql()`, `gate.leaks(reply, user_texts) -> [names]` (bounce and regenerate), `gate.after_reply(text) -> text without the rung marker` (logs `rung`). diff --git a/leetcode_coach/leetcode_coach/req_check.py b/leetcode_coach/leetcode_coach/req_check.py index f72eb17..3adcbd6 100644 --- a/leetcode_coach/leetcode_coach/req_check.py +++ b/leetcode_coach/leetcode_coach/req_check.py @@ -19,6 +19,7 @@ why the answer is extracted and validated rather than parsed whole. from __future__ import annotations from dataclasses import dataclass +import inspect import json import logging from typing import TYPE_CHECKING, Any @@ -65,6 +66,21 @@ CODE_SYSTEM = ( ) +CHECK_SCHEMA = { + "type": "object", + "properties": { + "verdict": {"enum": ["pass", "incomplete"]}, + "missing": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["verdict", "missing"], +} +CODE_SCHEMA = { + "type": "object", + "properties": {"code": {"type": "boolean"}}, + "required": ["code"], +} + + @dataclass(frozen=True) class Check: """A checker answer: verdict, the missing properties, and who answered.""" @@ -115,21 +131,63 @@ def check_prompt(statement: str, req: str, box: Toolbox) -> str: ) +def _takes_schema(ask: Callable[..., Any]) -> bool: + """A structured ``(system, prompt, schema)`` model function, not the CLI's.""" + try: + return "schema" in inspect.signature(ask).parameters + except TypeError, ValueError: + return False + + +def _structured[T]( + ask: Callable[..., dict[str, Any]], + system: str, + prompt: str, + schema: dict[str, Any], + validate: Callable[[dict[str, Any] | None], T | None], +) -> T | None: + try: + return validate(ask(system, prompt, schema)) + except Exception: # pylint: disable=broad-exception-caught + logging.getLogger(__name__).exception("model function could not answer") + return None + + +def _claude[T]( + ask: Callable[..., dict[str, Any]] | None, + system: str, + prompt: str, + validate: Callable[[dict[str, Any] | None], T | None], +) -> T | None: + asker = claude_cli.ask if ask is None else ask + reply = asker( + [*HAIKU_ARGS, "--system-prompt", system], prompt, timeout=HAIKU_TIMEOUT_S + ) + if claude_cli.is_limit_hit(reply): + return None + return validate(extract_json(str(reply.get("result", "")))) + + def ask_models[T]( system: str, prompt: str, validate: Callable[[dict[str, Any] | None], T | None], ask: Callable[..., dict[str, Any]] | None, + schema: dict[str, Any] | None = None, ) -> tuple[T, str] | None: - """Haiku, then Ollama; the first validated answer and who gave it.""" - asker = claude_cli.ask if ask is None else ask - reply = asker( - [*HAIKU_ARGS, "--system-prompt", system], prompt, timeout=HAIKU_TIMEOUT_S - ) - if not claude_cli.is_limit_hit(reply): - got = validate(extract_json(str(reply.get("result", "")))) - if got is not None: - return got, "haiku" + """The first validated answer and who gave it, then Ollama. + + `ask` is the CLI-style ``(args, prompt, **kw)`` (default: haiku) or a + structured ``(system, prompt, schema) -> dict`` model function. + """ + if ask is not None and _takes_schema(ask): + got = _structured(ask, system, prompt, schema or {"type": "object"}, validate) + via = "model" + else: + got = _claude(ask, system, prompt, validate) + via = "haiku" + if got is not None: + return got, via try: local = local_backend.chat( [ @@ -160,6 +218,7 @@ def check( check_prompt(statement, req, box), lambda obj: parse_verdict(obj, box.properties), ask, + CHECK_SCHEMA, ) if got is None: return Check("unverified") @@ -176,5 +235,5 @@ def looks_like_code( block: str, ask: Callable[..., dict[str, Any]] | None = None ) -> bool: """The model's call on a block the regex could not decide; prose if unsure.""" - got = ask_models(CODE_SYSTEM, block, _code_flag, ask) + got = ask_models(CODE_SYSTEM, block, _code_flag, ask, CODE_SCHEMA) return False if got is None else got[0] diff --git a/leetcode_coach/leetcode_coach/tests/test_req_check.py b/leetcode_coach/leetcode_coach/tests/test_req_check.py index e6aca83..621b962 100644 --- a/leetcode_coach/leetcode_coach/tests/test_req_check.py +++ b/leetcode_coach/leetcode_coach/tests/test_req_check.py @@ -164,3 +164,73 @@ def test_model_code_classifier_fails_open_to_not_code( ) -> None: _ollama(monkeypatch, ollama) assert req_check.looks_like_code("x ^ y", _haiku(haiku_says)) is is_code + + +class _Model: + """The tutor's ``ask_model(system, prompt, schema)``: a parsed dict, or a raise.""" + + def __init__(self, answer: dict[str, Any] | Exception) -> None: + self.answer = answer + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + + def __call__( + self, system: str, prompt: str, schema: dict[str, Any] + ) -> dict[str, Any]: + self.calls.append((system, prompt, schema)) + if isinstance(self.answer, Exception): + raise self.answer + return self.answer + + +def test_a_structured_model_function_answers_the_check( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sent = _ollama(monkeypatch, "unused") + ask = _Model({"verdict": "incomplete", "missing": ["constant-space", "XOR"]}) + got = req_check.check(STATEMENT, "req: loop", toolbox.load_safe(), ask) + assert got == req_check.Check("incomplete", ("constant-space",), "model") + system, prompt, schema = ask.calls[0] + assert system == req_check.CHECK_SYSTEM + assert "req: loop" in prompt + assert schema == req_check.CHECK_SCHEMA + assert sent == [] + + +@pytest.mark.parametrize( + "answer", + [RuntimeError("model down"), {"verdict": "incomplete", "missing": ["XOR"]}, {}], +) +def test_ollama_steps_in_when_the_model_function_cannot_answer( + monkeypatch: pytest.MonkeyPatch, answer: dict[str, Any] | Exception +) -> None: + _ollama(monkeypatch, '{"verdict": "pass", "missing": []}') + got = req_check.check(STATEMENT, "req: x", toolbox.load_safe(), _Model(answer)) + assert got == req_check.Check("pass", (), "ollama") + + +def test_a_structured_model_function_classifies_code() -> None: + ask = _Model({"code": True}) + assert req_check.looks_like_code("x ^= i", ask) is True + assert ask.calls[0][2] == req_check.CODE_SCHEMA + + +def test_a_model_function_with_no_schema_gets_a_bare_object_schema() -> None: + ask = _Model({"ok": 1}) + got = req_check.ask_models("sys", "prompt", lambda obj: obj, ask) + assert got == ({"ok": 1}, "model") + assert ask.calls[0][2] == {"type": "object"} + + +def test_a_callable_without_a_signature_is_the_cli_style( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _ollama(monkeypatch, "unused") + monkeypatch.setattr(req_check.inspect, "signature", _no_signature) + got = req_check.check( + STATEMENT, "req: x", toolbox.load_safe(), _haiku({"verdict": "pass"}) + ) + assert got.via == "haiku" + + +def _no_signature(_fn: object) -> None: + raise ValueError diff --git a/leetcode_coach/pyproject.toml b/leetcode_coach/pyproject.toml index a1472ca..a5005b9 100644 --- a/leetcode_coach/pyproject.toml +++ b/leetcode_coach/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "leetcode-coach" -version = "0.1.0" +version = "0.1.1" description = "The /leetcode coach as a library: phase gate, leak and stop gates, session log, grading, cards, relay" requires-python = ">=3.14" # Everything below loads lazily (rtdb, make_cards, relay.__main__): importing