mirror of
https://github.com/kuhyx/utils.git
synced 2026-11-20 18:52:30 +01:00
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WzxdWWTVKrxg1PGVP4wc6W
240 lines
8.0 KiB
Python
240 lines
8.0 KiB
Python
"""Judge kuhy's requirement list, and ambiguous code, on a cheap model.
|
|
|
|
DOCS-coach-spec.md decision 5: a separate ``claude -p --model haiku`` call (the
|
|
coaching session stays on opus), Ollama through `local_backend` when haiku
|
|
is out of quota, errors or answers something that is not the JSON asked for,
|
|
and when neither answers the gate fails OPEN: ``unverified``, kuhy continues.
|
|
|
|
The checker may only name *missing properties* from the toolbox vocabulary.
|
|
That is enforced here, not trusted to the prompt: every ``missing`` item that
|
|
is not a vocabulary key is dropped (an entry name like "XOR" is a leak, and an
|
|
invented property is noise). An ``incomplete`` whose every item was dropped
|
|
names nothing kuhy could fix, so it counts as no answer and falls through.
|
|
|
|
The haiku call runs with no tools, no MCP, no hooks and no session file: the
|
|
global CLAUDE.md still loads (no OAuth-compatible way to skip it), which is
|
|
why the answer is extracted and validated rather than parsed whole.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import inspect
|
|
import json
|
|
import logging
|
|
from typing import TYPE_CHECKING, Any
|
|
import urllib.error
|
|
|
|
from leetcode_coach import claude_cli, local_backend
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable
|
|
|
|
from leetcode_coach.toolbox import Toolbox
|
|
|
|
_LOG = logging.getLogger(__name__)
|
|
|
|
HAIKU_TIMEOUT_S = 50.0
|
|
LOCAL_TIMEOUT_S = 200.0
|
|
"""Ollama's share of a check: with haiku's 50 s it fits the relay's 280 s cap."""
|
|
HAIKU_ARGS = [
|
|
"--model",
|
|
"haiku",
|
|
"--tools",
|
|
"",
|
|
"--strict-mcp-config",
|
|
"--no-session-persistence",
|
|
"--disable-slash-commands",
|
|
"--settings",
|
|
'{"disableAllHooks": true}',
|
|
]
|
|
CHECK_SYSTEM = (
|
|
"You check a student's requirement list for a programming problem. You are "
|
|
"given the problem, the list, and a vocabulary of properties an operation "
|
|
"can have. Decide which vocabulary properties the operation that solves "
|
|
"the problem must have, and whether the list states each of them in any "
|
|
"words. Reply with ONE JSON object and nothing else: "
|
|
'{"verdict": "pass" | "incomplete", "missing": [<vocabulary property names>]}. '
|
|
"Never name an operation, a data structure or an algorithm; only the "
|
|
"property names from the vocabulary may appear in missing."
|
|
)
|
|
CODE_SYSTEM = (
|
|
"You decide whether a chat message is program source code (a line or "
|
|
"block someone would paste into a solution) or prose (a question or an "
|
|
'idea, even one that mentions a symbol). Reply with ONE JSON object: {"code": '
|
|
'true} or {"code": false}.'
|
|
)
|
|
|
|
|
|
CHECK_SCHEMA = {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": ["pass", "incomplete"]},
|
|
"missing": {"type": "array", "items": {"type": "string"}},
|
|
},
|
|
"required": ["verdict", "missing"],
|
|
}
|
|
CODE_SCHEMA = {
|
|
"type": "object",
|
|
"properties": {"code": {"type": "boolean"}},
|
|
"required": ["code"],
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Check:
|
|
"""A checker answer: verdict, the missing properties, and who answered."""
|
|
|
|
verdict: str
|
|
missing: tuple[str, ...] = ()
|
|
via: str = "none"
|
|
|
|
|
|
def extract_json(text: str) -> dict[str, Any] | None:
|
|
"""The JSON object starting at the first ``{`` (fences and chatter ignored)."""
|
|
start = text.find("{")
|
|
if start == -1:
|
|
return None
|
|
try:
|
|
obj, _end = json.JSONDecoder().raw_decode(text, start)
|
|
except ValueError as exc:
|
|
_LOG.warning("checker answer is not a JSON object: %s", exc)
|
|
return None
|
|
return obj if isinstance(obj, dict) else None
|
|
|
|
|
|
def parse_verdict(obj: object, vocabulary: dict[str, str]) -> Check | None:
|
|
"""A validated verdict, or None when the answer is unusable."""
|
|
if not isinstance(obj, dict) or obj.get("verdict") not in {"pass", "incomplete"}:
|
|
return None
|
|
missing = obj.get("missing", [])
|
|
if not isinstance(missing, list):
|
|
return None
|
|
kept = tuple(m for m in missing if isinstance(m, str) and m in vocabulary)
|
|
if obj["verdict"] == "pass":
|
|
return Check("pass")
|
|
return Check("incomplete", kept) if kept else None
|
|
|
|
|
|
def check_prompt(statement: str, req: str, box: Toolbox) -> str:
|
|
"""The checker's user message: problem, list, vocabulary, toolbox."""
|
|
vocab = "\n".join(
|
|
f"- {name}: {meaning}" for name, meaning in box.properties.items()
|
|
)
|
|
toolbox = "\n".join(
|
|
f"- {', '.join(e.properties)}" for e in box.entries if e.properties
|
|
)
|
|
return (
|
|
f"## Problem\n{statement.strip()}\n\n## Requirement list\n{req.strip()}\n\n"
|
|
f"## Property vocabulary\n{vocab}\n\n"
|
|
f"## Property sets the known operations offer\n{toolbox}\n"
|
|
)
|
|
|
|
|
|
def _takes_schema(ask: Callable[..., Any]) -> bool:
|
|
"""A structured ``(system, prompt, schema)`` model function, not the CLI's."""
|
|
try:
|
|
return "schema" in inspect.signature(ask).parameters
|
|
except TypeError, ValueError:
|
|
return False
|
|
|
|
|
|
def _structured[T](
|
|
ask: Callable[..., dict[str, Any]],
|
|
system: str,
|
|
prompt: str,
|
|
schema: dict[str, Any],
|
|
validate: Callable[[dict[str, Any] | None], T | None],
|
|
) -> T | None:
|
|
try:
|
|
return validate(ask(system, prompt, schema))
|
|
except Exception: # pylint: disable=broad-exception-caught
|
|
logging.getLogger(__name__).exception("model function could not answer")
|
|
return None
|
|
|
|
|
|
def _claude[T](
|
|
ask: Callable[..., dict[str, Any]] | None,
|
|
system: str,
|
|
prompt: str,
|
|
validate: Callable[[dict[str, Any] | None], T | None],
|
|
) -> T | None:
|
|
asker = claude_cli.ask if ask is None else ask
|
|
reply = asker(
|
|
[*HAIKU_ARGS, "--system-prompt", system], prompt, timeout=HAIKU_TIMEOUT_S
|
|
)
|
|
if claude_cli.is_limit_hit(reply):
|
|
return None
|
|
return validate(extract_json(str(reply.get("result", ""))))
|
|
|
|
|
|
def ask_models[T](
|
|
system: str,
|
|
prompt: str,
|
|
validate: Callable[[dict[str, Any] | None], T | None],
|
|
ask: Callable[..., dict[str, Any]] | None,
|
|
schema: dict[str, Any] | None = None,
|
|
) -> tuple[T, str] | None:
|
|
"""The first validated answer and who gave it, then Ollama.
|
|
|
|
`ask` is the CLI-style ``(args, prompt, **kw)`` (default: haiku) or a
|
|
structured ``(system, prompt, schema) -> dict`` model function.
|
|
"""
|
|
if ask is not None and _takes_schema(ask):
|
|
got = _structured(ask, system, prompt, schema or {"type": "object"}, validate)
|
|
via = "model"
|
|
else:
|
|
got = _claude(ask, system, prompt, validate)
|
|
via = "haiku"
|
|
if got is not None:
|
|
return got, via
|
|
try:
|
|
local = local_backend.chat(
|
|
[
|
|
{"role": "system", "content": system},
|
|
{"role": "user", "content": prompt},
|
|
],
|
|
timeout=LOCAL_TIMEOUT_S,
|
|
)
|
|
except (urllib.error.URLError, OSError, local_backend.ContextOverflowError) as exc:
|
|
_LOG.warning("Ollama could not answer the check either: %s", exc)
|
|
return None
|
|
got = validate(extract_json(local.text))
|
|
return None if got is None else (got, "ollama")
|
|
|
|
|
|
def check(
|
|
statement: str,
|
|
req: str,
|
|
box: Toolbox,
|
|
ask: Callable[..., dict[str, Any]] | None = None,
|
|
) -> Check:
|
|
"""Judge `req` against the problem; ``unverified`` when nobody can."""
|
|
if not box.properties:
|
|
_LOG.warning("no property vocabulary: the req list stays unverified")
|
|
return Check("unverified")
|
|
got = ask_models(
|
|
CHECK_SYSTEM,
|
|
check_prompt(statement, req, box),
|
|
lambda obj: parse_verdict(obj, box.properties),
|
|
ask,
|
|
CHECK_SCHEMA,
|
|
)
|
|
if got is None:
|
|
return Check("unverified")
|
|
verdict, via = got
|
|
return Check(verdict.verdict, verdict.missing, via)
|
|
|
|
|
|
def _code_flag(obj: dict[str, Any] | None) -> bool | None:
|
|
flag = None if obj is None else obj.get("code")
|
|
return flag if isinstance(flag, bool) else None
|
|
|
|
|
|
def looks_like_code(
|
|
block: str, ask: Callable[..., dict[str, Any]] | None = None
|
|
) -> bool:
|
|
"""The model's call on a block the regex could not decide; prose if unsure."""
|
|
got = ask_models(CODE_SYSTEM, block, _code_flag, ask, CODE_SCHEMA)
|
|
return False if got is None else got[0]
|