Add code_runner sandboxed example runner

Runs a solution (python3, javascript, typescript) against statement
examples in bubblewrap: no network, read-only root, scratch dir,
rlimits and a per-example timeout.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WzxdWYGW5NZL5uVpnqmNKo

.github/workflows/code-runner-precommit.yml

@@ -0,0 +1,73 @@
+name: code-runner pre-commit
+
+on:
+  push:
+    branches: [main]
+    # The workflow's own file counts as code_runner: editing only this file would
+    # otherwise trigger nothing, so a fix to the job itself could never prove
+    # itself.
+    paths: ['code_runner/**', '.github/workflows/code-runner-precommit.yml']
+  pull_request:
+    branches: [main]
+    paths: ['code_runner/**', '.github/workflows/code-runner-precommit.yml']
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      # Only pre-commit itself. Every linting hook runs in its own isolated
+      # env -- ruff, mypy, pylint, bandit and codespell each come from their
+      # own repo+rev, and the local hooks need nothing beyond bash and
+      # python3 -- so installing the full requirements.txt spent 9s a run on
+      # packages no pre-commit-stage hook imports. Verified by running all
+      # three configs green in a venv holding pre-commit and nothing else.
+      # The pin is read out of requirements.txt rather than repeated here, so
+      # it cannot drift from what the dependency-freshness gate governs.
+      - name: Install pre-commit
+        run: |
+          pip install "$(grep '^pre-commit==' code_runner/requirements.txt)"
+
+      # Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
+      # hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
+      # package jobs shared one key, raced to overwrite it with hook sets built
+      # from four different configs, and editing a per-package config did not
+      # invalidate it. The cache was therefore silently wrong rather than
+      # missing, and the hooks reinstalled from scratch on every run (38-49s).
+      - uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: >-
+            pre-commit-code-runner-${{ env.pythonLocation }}-${{
+            hashFiles('code_runner/.pre-commit-config.yaml') }}
+          # Prefix fallback, or this fix would evaporate on the very
+          # event that motivated it: bumping a hook rev changes the
+          # exact key, and a total miss means reinstalling every hook
+          # again. pre-commit's store is keyed internally by hook
+          # repo+rev, so a superseded entry is ignored disk rather
+          # than a wrong hook, and cache still re-saves under the new
+          # exact key after a prefix-only hit.
+          restore-keys: pre-commit-code-runner-${{ env.pythonLocation }}-
+      # mypy re-analyses the stdlib and every third-party stub from scratch
+      # without this, which measured 10s of the hook step in CI against 0.25s
+      # warm locally -- a 19MB cache dir does the whole difference. Keyed on
+      # the config, not the sources: mypy's own cache metadata records the
+      # interpreter, the flag set and a hash per source file, so it re-checks
+      # what changed and cannot pass stale code. Deliberately NOT keyed on
+      # github.sha -- that re-saves 19MB x3 every commit and would evict the
+      # 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
+      # the expensive part, the dependency analysis, which is what costs 10s.
+      - uses: actions/cache@v4
+        with:
+          path: .mypy_cache
+          key: >-
+            mypy-code-runner-${{ env.pythonLocation }}-${{
+            hashFiles('code_runner/.pre-commit-config.yaml') }}
+          restore-keys: mypy-code-runner-${{ env.pythonLocation }}-
+      - name: Run the hooks
+        run: >-
+          pre-commit run --config code_runner/.pre-commit-config.yaml
+          --all-files --show-diff-on-failure

.github/workflows/code-runner-tests.yml

@@ -0,0 +1,34 @@
+name: code-runner tests
+
+on:
+  push:
+    branches: [main]
+    # The workflow's own file counts as code_runner: editing only this file would
+    # otherwise trigger nothing, so a fix to the job itself could never prove
+    # itself.
+    paths: ['code_runner/**', '.github/workflows/code-runner-tests.yml']
+  pull_request:
+    branches: [main]
+    paths: ['code_runner/**', '.github/workflows/code-runner-tests.yml']
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: code_runner
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+          cache: pip
+          cache-dependency-path: code_runner/requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt && pip install --no-deps -e .
+
+      - name: Run tests with coverage
+        run: python -m pytest -v

code_runner/.pre-commit-config.yaml

@@ -0,0 +1,143 @@
+# Pre-commit Configuration for code_runner
+# Install:  pre-commit install && pre-commit install --hook-type pre-push
+# Run:      pre-commit run --all-files
+# Update:   pre-commit autoupdate
+
+# NOTE: this subproject lives inside the ~/src/utils monorepo, whose git root is
+# one level up. pre-commit always runs hooks from the git root, so every path
+# below is written relative to THAT, not to this directory, and the top-level
+# `files` key keeps these hooks off the sibling subprojects. CI invokes this
+# config exactly the same way:
+#   pre-commit run --config code_runner/.pre-commit-config.yaml --all-files
+files: ^code_runner/
+
+default_language_version:
+  python: python3
+default_stages: [pre-commit]
+fail_fast: false
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+      - id: check-toml
+      - id: check-added-large-files
+        args: [--maxkb=2000]
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: debug-statements
+      - id: name-tests-test
+        args: [--pytest-test-first]
+      - id: check-ast
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: requirements-txt-fixer
+
+  - repo: local
+    hooks:
+      - id: file-length-cap
+        name: file length <= 250 lines
+        entry: bash scripts/check_file_length.sh
+        language: system
+        pass_filenames: true
+      - id: subproject-scope
+        name: subproject scope regex still matches
+        entry: scripts/check_subproject_scope.sh code_runner
+        language: script
+        # always_run so the top-level `files` filter cannot skip the one hook
+        # whose job is to notice that filter has stopped matching anything.
+        always_run: true
+        pass_filenames: false
+      - id: no-noqa
+        name: Block noqa comments
+        entry: '(?i)#\s*(noqa|type:\s*ignore)'
+        language: pygrep
+        types: [python]
+      - id: no-ruff-noqa
+        name: Block ruff noqa file-level comments
+        entry: '(?i)#\s*ruff:\s*noqa'
+        language: pygrep
+        types: [python]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.17.2
+    hooks:
+      - id: ruff
+        args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix, --show-fixes]
+        types_or: [python, pyi]
+      - id: ruff-format
+        types_or: [python, pyi]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.13.0
+    hooks:
+      - id: mypy
+        args:
+          - --ignore-missing-imports
+          - --no-error-summary
+          - --disable-error-code=no-untyped-def
+          - --disable-error-code=no-untyped-call
+          - --disable-error-code=var-annotated
+          - --disable-error-code=no-any-unimported
+          - --disable-error-code=type-arg
+          - --disable-error-code=no-any-return
+          - --disable-error-code=misc
+          - --disable-error-code=unused-ignore
+          - --disable-error-code=unreachable
+          - --disable-error-code=assignment
+          - --disable-error-code=no-redef
+          - --disable-error-code=attr-defined
+          - --disable-error-code=arg-type
+          - --disable-error-code=union-attr
+          - --disable-error-code=call-overload
+          - --disable-error-code=return-value
+          - --disable-error-code=redundant-cast
+          - --disable-error-code=empty-body
+          - --disable-error-code=list-item
+
+  - repo: https://github.com/pylint-dev/pylint
+    rev: v4.0.8
+    hooks:
+      - id: pylint
+        args:
+          - --rcfile=code_runner/pyproject.toml
+          - --fail-under=10
+          - --jobs=4
+        additional_dependencies:
+          - pytest
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.9.4
+    hooks:
+      - id: bandit
+        args:
+          - -c
+          - code_runner/pyproject.toml
+          - --severity-level=high
+          - --confidence-level=medium
+          - --skip=B113
+        additional_dependencies: ["bandit[toml]"]
+        exclude: ^(code_runner/code_runner/tests/|.*test.*\.py$)
+
+  - repo: local
+    hooks:
+      - id: pytest-coverage
+        name: pytest with coverage enforcement
+        entry: scripts/run_subproject_tests.sh code_runner
+        language: script
+        types: [python]
+        pass_filenames: false
+        require_serial: true
+        stages: [pre-push]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.3.0
+    hooks:
+      - id: codespell
+        args:
+          - --skip=*.json,*.lock,.git,__pycache__,.venv

code_runner/README.md

@@ -0,0 +1,12 @@
+# code_runner
+
+Runs a learner's solution (Python 3, JavaScript, TypeScript) against a problem statement's examples inside bubblewrap: no network, read-only `/usr`, scratch dir, CPU/memory/process/file/time limits.
+
+```python
+import code_runner as cr
+
+statement = ...  # markdown from ~/.claude/skills/leetcode/fetch_problem.py
+results = cr.run("python3", code, cr.extract_starter(statement), cr.parse_examples(statement))
+```
+
+`run(lang, code, starter, examples) -> list[CaseResult]` with `n, input, expected, actual, ok, stdout, error, ms`. One sandbox process per example. TypeScript runs through node's type stripping (no type check, no enums). Needs `bwrap`, `prlimit`, `/usr/bin/python3`, node >= 22.18. Install: `./install.sh`.

code_runner/code_runner/__init__.py

@@ -0,0 +1,20 @@
+"""Run a learner's solution against a problem's examples in a bubblewrap sandbox."""
+
+from __future__ import annotations
+
+from code_runner._examples import parse_examples
+from code_runner._model import CaseResult, Example, Limits, RunnerError
+from code_runner._run import run
+from code_runner._starter import Entry, extract_starter, parse_starter
+
+__all__ = [
+    "CaseResult",
+    "Entry",
+    "Example",
+    "Limits",
+    "RunnerError",
+    "extract_starter",
+    "parse_examples",
+    "parse_starter",
+    "run",
+]

code_runner/code_runner/_compare.py

@@ -0,0 +1,66 @@
+"""Output comparison: exact JSON equality, float tolerance, order rules."""
+
+from __future__ import annotations
+
+import json
+import math
+
+TOLERANCE = 1e-5  # LeetCode accepts floating answers within 1e-5
+
+
+def _as_number(value: object) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, int | float):
+        return None
+    return value
+
+
+def _key(value: object) -> str:
+    return json.dumps(value, sort_keys=True, ensure_ascii=False)
+
+
+def _deep_sorted(value: object) -> object:
+    if isinstance(value, list):
+        return sorted((_deep_sorted(item) for item in value), key=_key)
+    return value
+
+
+def _equal(expected: object, actual: object) -> bool:
+    want, got = _as_number(expected), _as_number(actual)
+    if want is not None and got is not None:
+        return math.isclose(want, got, rel_tol=TOLERANCE, abs_tol=TOLERANCE)
+    if isinstance(expected, list) and isinstance(actual, list):
+        return len(expected) == len(actual) and all(
+            _equal(e, a) for e, a in zip(expected, actual, strict=True)
+        )
+    if isinstance(expected, dict) and isinstance(actual, dict):
+        return expected.keys() == actual.keys() and all(
+            _equal(expected[k], actual[k]) for k in expected
+        )
+    return type(expected) is type(actual) and expected == actual
+
+
+def _top_sorted(value: object) -> object:
+    if isinstance(value, list):
+        return sorted(value, key=_key)
+    return value
+
+
+def matches(expected: object, actual: object, order: str) -> bool:
+    """Whether ``actual`` is an accepted answer for ``expected``.
+
+    Args:
+        expected: the statement's answer (JSON value).
+        actual: what the solution returned (JSON value).
+        order: "exact", "any" (list order free) or "deep" (nested order free too).
+
+    Raises:
+        ValueError: ``order`` is none of those.
+    """
+    if order == "exact":
+        return _equal(expected, actual)
+    if order == "any":
+        return _equal(_top_sorted(expected), _top_sorted(actual))
+    if order == "deep":
+        return _equal(_deep_sorted(expected), _deep_sorted(actual))
+    msg = f"unknown order {order!r}"
+    raise ValueError(msg)

code_runner/code_runner/_examples.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import ast
+import json
+import logging
+import re
+from typing import Final
+
+from code_runner._model import Example, Order
+from code_runner._split import split_top_level
+
+_logger: Final = logging.getLogger(__name__)
+_LABEL: Final = re.compile(r"^(Input|Output)\s*:\s*(.*)$")
+_PAIR: Final = re.compile(r"^([A-Za-z_]\w*)\s*=\s*(.+)$", re.DOTALL)
+_ANY_ORDER: Final = re.compile(r"any order|order of the output", re.IGNORECASE)
+_DEEP_ORDER: Final = re.compile(
+    r"order of the (?:triplets|quadruplets|subsets|groups|pairs|elements)",
+    re.IGNORECASE,
+)
+
+
+def _value(text: str) -> object:
+    try:
+        return json.loads(text)
+    except ValueError:
+        try:
+            return ast.literal_eval(text)
+        except (ValueError, SyntaxError) as exc:
+            raise ValueError(text[:40]) from exc
+
+
+def _arguments(text: str) -> dict[str, object]:
+    args: dict[str, object] = {}
+    for part in split_top_level(text):
+        pair = _PAIR.match(part)
+        if pair is None:
+            msg = f"not a name = value pair: {part[:40]!r}"
+            raise ValueError(msg)
+        args[pair.group(1)] = _value(pair.group(2))
+    return args
+
+
+def _labels(body: str) -> list[tuple[str, str]]:
+    cleaned = (
+        line.replace("**", "").replace("`", "").strip() for line in body.splitlines()
+    )
+    matched = (_LABEL.match(line) for line in cleaned)
+    return [(m.group(1), m.group(2).strip()) for m in matched if m is not None]
+
+
+def _order(body: str) -> Order:
+    if _DEEP_ORDER.search(body):
+        return "deep"
+    return "any" if _ANY_ORDER.search(body) else "exact"
+
+
+def parse_examples(statement: str) -> list[Example]:
+    body = re.split(r"^## ", statement, maxsplit=1, flags=re.MULTILINE)[0]
+    order = _order(body)
+    found: list[Example] = []
+    pending: str | None = None
+    number = 0
+    for label, text in _labels(body):
+        if label == "Input":
+            pending = text
+            continue
+        if pending is None:
+            continue
+        number += 1
+        try:
+            found.append(Example(number, _arguments(pending), _value(text), order))
+        except ValueError as exc:
+            _logger.warning("example %d skipped, not runnable: %s", number, exc)
+        pending = None
+    return found

code_runner/code_runner/_languages.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from importlib import resources
+import json
+import os
+from pathlib import Path
+import shutil
+from typing import TYPE_CHECKING
+
+from code_runner._model import RunnerError
+from code_runner._sandbox import WORK
+
+if TYPE_CHECKING:
+    from code_runner._starter import Entry
+
+MEGABYTE = 1024 * 1024
+_NODE_FLAGS = (
+    "--permission",
+    f"--allow-fs-read={WORK}",
+    f"--allow-fs-write={WORK}",
+    "--max-old-space-size=128",
+)
+
+
+@dataclass(frozen=True)
+class Language:
+    name: str
+    solution_file: str
+    harness_file: str
+    program: str
+    binary_env: str
+    binary_default: str
+    flags: tuple[str, ...]
+    memory_bytes: int
+
+    def resolve(self) -> Path:
+        for candidate in (
+            os.environ.get(self.binary_env),
+            self.binary_default,
+            shutil.which(self.program),
+        ):
+            if candidate and Path(candidate).is_file():
+                return Path(candidate).resolve()
+        msg = f"{self.program} not found (set {self.binary_env})"
+        raise RunnerError(msg)
+
+    def command(self, binary: Path) -> list[str]:
+        return [str(binary), *self.flags, f"{WORK}/{self.harness_file}", WORK]
+
+    def ro_binds(self, binary: Path) -> tuple[str, ...]:
+        if binary.is_relative_to("/usr"):
+            return ()
+        return (str(binary.parent.parent),)
+
+    def prepare(
+        self, scratch: Path, code: str, entry: Entry, args: list[object]
+    ) -> None:
+        harness = resources.files("code_runner").joinpath("harness", self.harness_file)
+        (scratch / self.harness_file).write_text(harness.read_text(encoding="utf-8"))
+        text = code
+        if self.name == "typescript":
+            text += f"\nmodule.exports = {{ __entry: {entry.cls or entry.name} }};\n"
+        (scratch / self.solution_file).write_text(text, encoding="utf-8")
+        spec = {
+            "cls": entry.cls,
+            "fn": entry.name,
+            "args": args,
+            "typescript": self.name == "typescript",
+        }
+        (scratch / "input.json").write_text(json.dumps(spec), encoding="utf-8")
+
+
+PYTHON = Language(
+    "python3",
+    "solution.py",
+    "harness.py",
+    "python3",
+    "CODE_RUNNER_PYTHON",
+    "/usr/bin/python3",
+    ("-I", "-B"),
+    512 * MEGABYTE,
+)
+JAVASCRIPT = Language(
+    "javascript",
+    "solution.js",
+    "harness.js",
+    "node",
+    "CODE_RUNNER_NODE",
+    "/usr/bin/node",
+    _NODE_FLAGS,
+    2048 * MEGABYTE,
+)
+TYPESCRIPT = Language(
+    "typescript",
+    "solution.cts",
+    "harness.js",
+    "node",
+    "CODE_RUNNER_NODE",
+    "/usr/bin/node",
+    _NODE_FLAGS,
+    2048 * MEGABYTE,
+)
+_BY_NAME = {
+    "python3": PYTHON,
+    "python": PYTHON,
+    "javascript": JAVASCRIPT,
+    "js": JAVASCRIPT,
+    "typescript": TYPESCRIPT,
+    "ts": TYPESCRIPT,
+}
+
+
+def get_language(name: str) -> Language:
+    try:
+        return _BY_NAME[name.lower()]
+    except KeyError:
+        msg = f"unsupported language {name!r}; use python3, javascript or typescript"
+        raise RunnerError(msg) from None

code_runner/code_runner/_model.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+# "any": list order is free; "deep": the order inside its items is free too.
+Order = Literal["exact", "any", "deep"]
+MAX_CAPTURE = 64 * 1024
+
+
+class RunnerError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Example:
+    n: int
+    args: dict[str, object]
+    expected: object
+    order: Order = "exact"
+
+
+@dataclass(frozen=True)
+class CaseResult:
+    n: int
+    input: dict[str, object]
+    expected: object
+    actual: object
+    ok: bool
+    stdout: str = ""
+    error: str | None = None
+    ms: float = 0.0
+
+
+@dataclass(frozen=True)
+class Limits:
+    timeout_s: float = 2.0
+    nproc: int = 64
+    fsize_bytes: int = 1024 * 1024
+    nofile: int = 256

code_runner/code_runner/_run.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+import json
+import os
+from pathlib import Path
+import stat
+import tempfile
+
+from code_runner._compare import matches
+from code_runner._languages import Language, get_language
+from code_runner._model import MAX_CAPTURE, CaseResult, Example, Limits
+from code_runner._sandbox import BoxResult, execute
+from code_runner._starter import Entry, parse_starter
+
+Box = Callable[[list[str], Path, Limits, int, tuple[str, ...]], BoxResult]
+_OUT_OF_MEMORY = ("out of memory", "allocation failed")
+
+
+def _positional(example: Example, entry: Entry) -> list[object]:
+    if set(example.args) == set(entry.params):
+        return [example.args[name] for name in entry.params]
+    if len(example.args) == len(entry.params):
+        return list(example.args.values())
+    given, wanted = len(example.args), len(entry.params)
+    msg = f"ArgumentError: example gives {given} arguments, {entry.name} takes {wanted}"
+    raise ValueError(msg)
+
+
+def _read_result(scratch: Path) -> dict[str, object] | None:
+    # The scratch dir is the solution's: never follow a link it planted there.
+    flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
+    try:
+        fd = os.open(scratch / "result.json", flags)
+    except OSError:
+        return None
+    with os.fdopen(fd, "rb") as handle:
+        if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
+            return None
+        raw = handle.read(MAX_CAPTURE * 16)
+    try:
+        payload = json.loads(raw)
+    except ValueError:
+        return None
+    return payload if isinstance(payload, dict) and "ok" in payload else None
+
+
+def _crash(box: BoxResult) -> str:
+    low = box.stderr.lower()
+    if any(marker in low for marker in _OUT_OF_MEMORY):
+        return "MemoryLimitExceeded"
+    return f"Crashed (exit {box.returncode})"
+
+
+def _outcome(scratch: Path, box: BoxResult) -> tuple[object, str | None, float]:
+    if box.timed_out:
+        return None, "TimeLimitExceeded", box.elapsed_ms
+    payload = _read_result(scratch)
+    if payload is None:
+        return None, _crash(box), box.elapsed_ms
+    ms = payload.get("ms")
+    took = float(ms) if isinstance(ms, int | float) else box.elapsed_ms
+    if payload["ok"] is True:
+        return payload.get("value"), None, took
+    return None, str(payload.get("error", "Error")), took
+
+
+@dataclass(frozen=True)
+class _Job:
+    language: Language
+    binary: Path
+    code: str
+    entry: Entry
+    limits: Limits
+    box: Box
+
+
+def _case(job: _Job, example: Example) -> CaseResult:
+    try:
+        args = _positional(example, job.entry)
+    except ValueError as exc:
+        return CaseResult(
+            example.n, example.args, example.expected, None, ok=False, error=str(exc)
+        )
+    language = job.language
+    with tempfile.TemporaryDirectory(prefix="code-runner-") as tmp:
+        scratch = Path(tmp)
+        language.prepare(scratch, job.code, job.entry, args)
+        done = job.box(
+            language.command(job.binary),
+            scratch,
+            job.limits,
+            language.memory_bytes,
+            language.ro_binds(job.binary),
+        )
+        actual, error, ms = _outcome(scratch, done)
+    ok = error is None and matches(example.expected, actual, example.order)
+    return CaseResult(
+        example.n, example.args, example.expected, actual, ok, done.stdout, error, ms
+    )
+
+
+def run(
+    lang: str,
+    code: str,
+    starter: str,
+    examples: Sequence[Example],
+    *,
+    limits: Limits | None = None,
+    box: Box = execute,
+) -> list[CaseResult]:
+    language = get_language(lang)
+    entry = parse_starter(language.name, starter)
+    job = _Job(language, language.resolve(), code, entry, limits or Limits(), box)
+    return [_case(job, example) for example in examples]

code_runner/code_runner/_sandbox.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import functools
+import math
+import os
+from pathlib import Path
+import signal
+import subprocess
+import tempfile
+import time
+from typing import IO
+
+from code_runner._model import MAX_CAPTURE, Limits, RunnerError
+
+WORK = "/work"
+PRLIMIT = "/usr/bin/prlimit"
+_ROOT_LINKS = ("bin", "sbin", "lib", "lib64")
+
+
+@dataclass(frozen=True)
+class BoxResult:
+    returncode: int
+    timed_out: bool
+    stdout: str
+    stderr: str
+    elapsed_ms: float
+
+
+def _root_links() -> list[str]:
+    args: list[str] = []
+    for name in _ROOT_LINKS:
+        path = Path("/") / name
+        if path.is_symlink():
+            args += ["--symlink", str(path.readlink()), str(path)]
+        elif path.is_dir():
+            args += ["--ro-bind", str(path), str(path)]
+    return args
+
+
+def _base_argv() -> list[str]:
+    return [
+        "bwrap",
+        "--unshare-all",  # no --share-net: no network at all, loopback included
+        "--die-with-parent",
+        "--new-session",
+        "--clearenv",
+        "--ro-bind",
+        "/usr",
+        "/usr",
+        *_root_links(),
+        "--proc",
+        "/proc",
+        "--dev",
+        "/dev",
+        "--tmpfs",
+        "/tmp",
+    ]
+
+
+def bwrap_argv(
+    command: list[str],
+    scratch: Path,
+    limits: Limits,
+    memory_bytes: int,
+    ro_binds: tuple[str, ...] = (),
+) -> list[str]:
+    argv = _base_argv()
+    for bind in ro_binds:
+        argv += ["--ro-bind", bind, bind]
+    argv += ["--bind", str(scratch), WORK, "--remount-ro", "/", "--chdir", WORK]
+    for name, value in (
+        ("HOME", WORK),
+        ("PATH", "/usr/bin"),
+        ("LANG", "C.UTF-8"),
+        ("PYTHONIOENCODING", "utf-8"),
+    ):
+        argv += ["--setenv", name, value]
+    cpu = math.ceil(limits.timeout_s) + 1
+    return [
+        *argv,
+        "--",
+        PRLIMIT,
+        f"--as={memory_bytes}",
+        f"--nproc={limits.nproc}",
+        f"--fsize={limits.fsize_bytes}",
+        f"--nofile={limits.nofile}",
+        f"--cpu={cpu}",
+        "--core=0",
+        "--",
+        *command,
+    ]
+
+
+@functools.cache
+def check_sandbox() -> None:
+    if not Path(PRLIMIT).is_file():
+        msg = f"{PRLIMIT} is missing (util-linux)"
+        raise RunnerError(msg)
+    try:
+        probe = subprocess.run(
+            [*_base_argv(), "/usr/bin/true"],
+            capture_output=True,
+            check=False,
+            timeout=20,
+        )
+    except OSError as exc:
+        msg = f"bubblewrap (bwrap) cannot start: {exc}"
+        raise RunnerError(msg) from exc
+    if probe.returncode != 0:
+        detail = probe.stderr.decode(errors="replace").strip()
+        msg = f"bubblewrap cannot create the sandbox: {detail}"
+        raise RunnerError(msg)
+
+
+def _read(handle: IO[bytes]) -> str:
+    handle.seek(0)
+    return handle.read(MAX_CAPTURE).decode("utf-8", errors="replace")
+
+
+def execute(
+    command: list[str],
+    scratch: Path,
+    limits: Limits,
+    memory_bytes: int,
+    ro_binds: tuple[str, ...] = (),
+) -> BoxResult:
+    check_sandbox()
+    argv = bwrap_argv(command, scratch, limits, memory_bytes, ro_binds)
+    with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
+        started = time.monotonic()
+        with subprocess.Popen(
+            argv,
+            stdin=subprocess.DEVNULL,
+            stdout=out,
+            stderr=err,
+            env={"PATH": "/usr/bin:/bin"},
+            start_new_session=True,
+        ) as proc:
+            timed_out = False
+            try:
+                proc.wait(timeout=limits.timeout_s)
+            except subprocess.TimeoutExpired:
+                timed_out = True
+                os.killpg(proc.pid, signal.SIGKILL)
+        elapsed = (time.monotonic() - started) * 1000
+        return BoxResult(proc.returncode, timed_out, _read(out), _read(err), elapsed)

code_runner/code_runner/_split.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+_OPEN = "([{<"
+_CLOSE = ")]}>"
+
+
+def split_top_level(text: str) -> list[str]:
+    parts: list[str] = []
+    depth = 0
+    quote = ""
+    escaped = False
+    start = 0
+    for i, ch in enumerate(text):
+        if escaped:
+            escaped = False
+        elif quote:
+            escaped = ch == "\\"
+            quote = "" if ch == quote else quote
+        elif ch in "\"'":
+            quote = ch
+        elif ch in _OPEN:
+            depth += 1
+        elif ch in _CLOSE and not (ch == ">" and text[i - 1] in "=-"):
+            depth = max(depth - 1, 0)
+        elif ch == "," and depth == 0:
+            parts.append(text[start:i])
+            start = i + 1
+    parts.append(text[start:])
+    return [part.strip() for part in parts]

code_runner/code_runner/_starter.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import re
+
+from code_runner._model import RunnerError
+from code_runner._split import split_top_level
+
+_IDENT = r"[A-Za-z_$][\w$]*"
+_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
+_LINE_COMMENT = re.compile(r"^\s*(?://|#).*$", re.MULTILINE)
+_PY_CLASS = re.compile(r"^class\s+(\w+)", re.MULTILINE)
+_PY_DEF = re.compile(r"\bdef\s+(?!__)(\w+)\s*\(")
+_PY_FUNC = re.compile(r"^def\s+(\w+)\s*\(", re.MULTILINE)
+_JS_CLASS = re.compile(rf"^\s*class\s+({_IDENT})", re.MULTILINE)
+_JS_METHOD = re.compile(
+    rf"^\s+(?:async\s+)?(?!constructor\b)({_IDENT})\s*\(", re.MULTILINE
+)
+_JS_ASSIGNED = re.compile(
+    rf"\b(?:var|let|const)\s+({_IDENT})\s*(?::[^=]+)?=\s*(?:async\s+)?(?:function\b\s*\w*\s*)?\("
+)
+_JS_DECLARED = re.compile(rf"\bfunction\s+({_IDENT})\s*\(")
+_NOT_METHODS = {"if", "for", "while", "switch", "catch", "return", "function"}
+
+
+@dataclass(frozen=True)
+class Entry:
+    cls: str | None
+    name: str
+    params: tuple[str, ...]
+
+
+def _no_entry(lang: str) -> RunnerError:
+    return RunnerError(f"no entry point found in the {lang} starter code")
+
+
+def _balanced(text: str, open_at: int) -> str:
+    depth = 0
+    for i in range(open_at, len(text)):
+        ch = text[i]
+        if ch in "([{" or (ch == "<" and depth > 0):
+            depth += 1
+        elif ch in ")]}" or (ch == ">" and text[i - 1] != "=" and depth > 1):
+            depth -= 1
+            if depth == 0:
+                return text[open_at + 1 : i]
+    return text[open_at + 1 :]
+
+
+def _params(code: str, open_at: int) -> tuple[str, ...]:
+    names: list[str] = []
+    for part in split_top_level(_balanced(code, open_at)):
+        found = re.match(_IDENT, part.strip())
+        if found:
+            names.append(found.group())
+    return tuple(names)
+
+
+def _python(text: str) -> Entry | None:
+    code = _LINE_COMMENT.sub("", text)
+    owner = _PY_CLASS.search(code)
+    if owner is None:
+        found = _PY_FUNC.search(code)
+        return found and Entry(None, found.group(1), _params(code, found.end() - 1))
+    found = _PY_DEF.search(code, owner.end())
+    if found is None:
+        return None
+    params = _params(code, found.end() - 1)
+    return Entry(
+        owner.group(1), found.group(1), tuple(p for p in params if p != "self")
+    )
+
+
+def _javascript(text: str) -> Entry | None:
+    code = _LINE_COMMENT.sub("", _BLOCK_COMMENT.sub("", text))
+    owner = _JS_CLASS.search(code)
+    if owner is not None:
+        for method in _JS_METHOD.finditer(code, owner.end()):
+            if method.group(1) not in _NOT_METHODS:
+                return Entry(
+                    owner.group(1), method.group(1), _params(code, method.end() - 1)
+                )
+        return None
+    found = _JS_ASSIGNED.search(code) or _JS_DECLARED.search(code)
+    return found and Entry(None, found.group(1), _params(code, found.end() - 1))
+
+
+def parse_starter(lang: str, text: str) -> Entry:
+    entry = _python(text) if lang == "python3" else _javascript(text)
+    if entry is None:
+        raise _no_entry(lang)
+    return entry
+
+
+def extract_starter(statement: str) -> str:
+    lines = statement.splitlines(keepends=True)
+    heads = [i for i, line in enumerate(lines) if line.startswith("## Starter code")]
+    if not heads:
+        return ""
+    body = lines[heads[0] + 1 :]
+    if body and body[0].startswith("```"):
+        body = body[1:]
+        end = next(
+            (i for i, line in enumerate(body) if line.startswith("```")), len(body)
+        )
+        return "".join(body[:end])
+    end = next((i for i, line in enumerate(body) if line.startswith("## ")), len(body))
+    return "".join(body[:end])

code_runner/code_runner/harness/harness.js

@@ -0,0 +1,45 @@
+"use strict";
+// Runs inside the sandbox: call the solution once, write result.json.
+const fs = require("node:fs");
+const path = require("node:path");
+
+const root = process.argv[2];
+const spec = JSON.parse(fs.readFileSync(path.join(root, "input.json"), "utf8"));
+const IDENT = /^[A-Za-z_$][\w$]*$/;
+
+function describe(e) {
+  const name = e && typeof e.name === "string" ? e.name : "Error";
+  const message = e && e.message !== undefined ? e.message : String(e);
+  return `${name}: ${message}`.slice(0, 300);
+}
+
+function writeResult(payload) {
+  // Unique name + rename: a symlink planted at result.json is replaced, never followed.
+  const unique = path.join(root, `.result-${process.pid}-${Math.random().toString(36).slice(2)}`);
+  fs.writeFileSync(unique, JSON.stringify(payload), { flag: "wx" });
+  fs.renameSync(unique, path.join(root, "result.json"));
+}
+
+function load(target) {
+  if (!IDENT.test(target)) throw new TypeError("bad entry name");
+  if (spec.typescript) return require(path.join(root, "solution.cts")).__entry;
+  const source = fs.readFileSync(path.join(root, "solution.js"), "utf8");
+  return new Function(`${source}\n;return ${target};`)();
+}
+
+function call() {
+  const entry = load(spec.cls || spec.fn);
+  if (spec.cls) return new entry()[spec.fn](...spec.args);
+  return entry(...spec.args);
+}
+
+const started = process.hrtime.bigint();
+let payload;
+try {
+  const value = call();
+  payload = { ok: true, value: value === undefined ? null : JSON.parse(JSON.stringify(value)) };
+} catch (e) {
+  payload = { ok: false, error: describe(e) };
+}
+payload.ms = Number(process.hrtime.bigint() - started) / 1e6;
+writeResult(payload);

code_runner/code_runner/harness/harness.py

@@ -0,0 +1,69 @@
+"""Runs inside the sandbox: call the solution once, write result.json."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import time
+from typing import Any
+
+PRELUDE = (
+    "from typing import *\n"
+    "import bisect, collections, functools, heapq, itertools, math\n"
+    "import operator, re, string\n"
+)
+MAX_MESSAGE = 300
+
+
+def describe(exc: BaseException) -> str:
+    return f"{type(exc).__name__}: {exc}"[:MAX_MESSAGE]
+
+
+def write_result(root: Path, payload: dict[str, object]) -> None:
+    # Unique name + rename: a symlink the solution planted at result.json is
+    # replaced by this file, never followed.
+    handle, name = tempfile.mkstemp(dir=root, prefix=".result-")
+    with os.fdopen(handle, "w", encoding="utf-8") as out:
+        json.dump(payload, out)
+    Path(name).replace(root / "result.json")
+
+
+def call(root: Path, spec: dict[str, Any]) -> object:
+    namespace: dict[str, Any] = {"__name__": "solution"}
+    exec(PRELUDE, namespace)
+    source = (root / "solution.py").read_text(encoding="utf-8")
+    exec(compile(source, "solution.py", "exec"), namespace)
+    owner, name = spec["cls"], str(spec["fn"])
+    if owner is None:
+        target = namespace.get(name)
+    else:
+        cls = namespace.get(str(owner))
+        target = getattr(cls() if cls else None, name, None)
+    if target is None:
+        message = f"name {owner or name!r} is not defined"
+        raise NameError(message)
+    return target(*spec["args"])
+
+
+def main(directory: str) -> None:
+    root = Path(directory)
+    spec = json.loads((root / "input.json").read_text(encoding="utf-8"))
+    started = time.perf_counter()
+    try:
+        value = call(root, spec)
+        payload: dict[str, object] = {
+            "ok": True,
+            "value": json.loads(json.dumps(value)),
+        }
+    except BaseException as exc:
+        payload = {"ok": False, "error": describe(exc)}
+    payload["ms"] = (time.perf_counter() - started) * 1000
+    sys.stdout.flush()
+    write_result(root, payload)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])

code_runner/code_runner/tests/conftest.py

@@ -0,0 +1,70 @@
+"""Shared fixtures: real-sandbox tests are skipped where bubblewrap cannot run."""
+
+from __future__ import annotations
+
+import shutil
+
+import pytest
+
+from code_runner._model import Example, RunnerError
+from code_runner._sandbox import check_sandbox
+
+
+def _bwrap_works() -> bool:
+    try:
+        check_sandbox()
+    except RunnerError:
+        return False
+    return True
+
+
+needs_sandbox = pytest.mark.skipif(not _bwrap_works(), reason="bwrap/userns unusable")
+needs_node = pytest.mark.skipif(
+    not _bwrap_works() or not shutil.which("node"), reason="node or bwrap missing"
+)
+
+TWO_SUM_STARTER = {
+    "python3": "class Solution:\n    def twoSum(self, nums: list[int], target: int) -> list[int]:\n",
+    "javascript": "var twoSum = function(nums, target) {\n\n};",
+    "typescript": "function twoSum(nums: number[], target: number): number[] {\n};",
+}
+TWO_SUM_CODE = {
+    "python3": (
+        "class Solution:\n"
+        "    def twoSum(self, nums, target):\n"
+        "        seen = {}\n"
+        "        for i, n in enumerate(nums):\n"
+        "            if target - n in seen:\n"
+        "                return [i, seen[target - n]]\n"
+        "            seen[n] = i\n"
+    ),
+    "javascript": (
+        "var twoSum = function(nums, target) {\n"
+        "  const seen = new Map();\n"
+        "  for (let i = 0; i < nums.length; i++) {\n"
+        "    if (seen.has(target - nums[i])) return [i, seen.get(target - nums[i])];\n"
+        "    seen.set(nums[i], i);\n"
+        "  }\n"
+        "};"
+    ),
+    "typescript": (
+        "function twoSum(nums: number[], target: number): number[] {\n"
+        "  const seen = new Map<number, number>();\n"
+        "  for (let i = 0; i < nums.length; i++) {\n"
+        "    const j = seen.get(target - nums[i]);\n"
+        "    if (j !== undefined) return [i, j];\n"
+        "    seen.set(nums[i], i);\n"
+        "  }\n"
+        "  return [];\n"
+        "}"
+    ),
+}
+
+
+@pytest.fixture
+def two_sum_examples() -> list[Example]:
+    return [
+        Example(1, {"nums": [2, 7, 11, 15], "target": 9}, [0, 1], "any"),
+        Example(2, {"nums": [3, 2, 4], "target": 6}, [1, 2], "any"),
+        Example(3, {"nums": [3, 3], "target": 6}, [0, 1], "any"),
+    ]

code_runner/code_runner/tests/test_compare.py

@@ -0,0 +1,42 @@
+import pytest
+
+from code_runner._compare import matches
+
+
+@pytest.mark.parametrize(
+    ("expected", "actual", "order", "want"),
+    [
+        ([0, 1], [0, 1], "exact", True),
+        ([0, 1], [1, 0], "exact", False),
+        ([0, 1], [1, 0], "any", True),
+        ([0, 1], [1, 0, 2], "any", False),
+        ([[1, 2], [3]], [[3], [1, 2]], "any", True),
+        ([[1, 2], [3]], [[3], [2, 1]], "any", False),
+        ([[1, 2], [3]], [[3], [2, 1]], "deep", True),
+        ([[1, 2], [3]], [[3], [2, 9]], "deep", False),
+        (3, 3.0, "exact", True),
+        (0.1 + 0.2, 0.3, "exact", True),
+        (2.0, 2.001, "exact", False),
+        (True, 1, "exact", False),
+        (1, True, "exact", False),
+        (True, True, "exact", True),
+        (None, None, "exact", True),
+        (None, 0, "exact", False),
+        ("ab", "ab", "exact", True),
+        ("ab", "ba", "exact", False),
+        ("1", 1, "exact", False),
+        ({"a": [1]}, {"a": [1]}, "exact", True),
+        ({"a": 1}, {"b": 1}, "exact", False),
+        ({"a": 1}, {"a": 2}, "exact", False),
+        (5, 5, "any", True),
+        ([1], 1, "any", False),
+        ([1.0, 2.0], [2.0, 1.0000001], "any", True),
+    ],
+)
+def test_matches(expected: object, actual: object, order: str, *, want: bool) -> None:
+    assert matches(expected, actual, order) is want
+
+
+def test_unknown_order_is_refused() -> None:
+    with pytest.raises(ValueError, match="order"):
+        matches(1, 1, "sideways")

code_runner/code_runner/tests/test_examples.py

@@ -0,0 +1,106 @@
+from code_runner._examples import parse_examples
+from code_runner._model import Example
+
+TWO_SUM = """# 1. Two Sum (Easy)
+Tags: Array, Hash Table
+
+Return indices.
+
+You can return the answer in any order.
+
+**Example 1:**
+
+**Input:** `nums = [2,7,11,15], target = 9`
+
+**Output:** `[0,1]`
+
+**Explanation:** 2 + 7.
+
+**Example 2:**
+
+**Input:** `nums = [3,2,4], target = 6`
+
+**Output:** `[1,2]`
+
+## Starter code (python3)
+```
+class Solution:
+    def twoSum(self, nums, target):
+```
+"""
+
+FENCED = """# 5. Pal
+
+Output order does not matter.
+
+**Example 1:**
+```
+Input: s = "a,b", flag = true
+Output: "b"
+Explanation: Output: not an example
+```
+**Example 2:**
+```
+Input: s = 'x', flag = false
+Output: None
+```
+"""
+
+THREE_SUM = """# 15. 3Sum
+Notice that the order of the output and the order of the triplets does not matter.
+**Example 1:**
+**Input:** nums = [-1,0,1,2,-1,-4]
+**Output:** [[-1,-1,2],[-1,0,1]]
+"""
+
+SKIPPED = """# 2. Add Two Numbers
+**Example 1:**
+**Input:** l1 = 1->2
+**Output:** [1,2]
+**Example 2:**
+**Input:** a = 1, b = 2
+**Output:** 3.00000
+**Example 3:**
+**Input:** a = 1
+**Output:** 1->2
+**Example 4:**
+**Input:** a
+**Output:** 1
+"""
+
+
+def test_two_sum_examples_are_typed_and_ordered_any() -> None:
+    found = parse_examples(TWO_SUM)
+    assert found == [
+        Example(1, {"nums": [2, 7, 11, 15], "target": 9}, [0, 1], "any"),
+        Example(2, {"nums": [3, 2, 4], "target": 6}, [1, 2], "any"),
+    ]
+
+
+def test_fenced_examples_and_python_literals() -> None:
+    found = parse_examples(FENCED)
+    assert [e.args for e in found] == [
+        {"s": "a,b", "flag": True},
+        {"s": "x", "flag": False},
+    ]
+    assert [e.expected for e in found] == ["b", None]
+    assert {e.order for e in found} == {"exact"}
+
+
+def test_triplets_sentence_means_deep_order() -> None:
+    assert parse_examples(THREE_SUM)[0].order == "deep"
+
+
+def test_unparseable_examples_are_skipped_but_numbering_is_kept() -> None:
+    found = parse_examples(SKIPPED)
+    assert found == [Example(2, {"a": 1, "b": 2}, 3.0, "exact")]
+
+
+def test_output_without_input_and_empty_statement() -> None:
+    assert parse_examples("**Output:** 1\n") == []
+    assert parse_examples("") == []
+
+
+def test_starter_section_is_never_scanned() -> None:
+    text = "# t\n## Starter code (python3)\n```\nInput: a = 1\nOutput: 2\n```\n"
+    assert parse_examples(text) == []

code_runner/code_runner/tests/test_internals.py

@@ -0,0 +1,192 @@
+"""Branches that a healthy sandbox never reaches: crashes, bad results, no bwrap."""
+
+from __future__ import annotations
+
+import dataclasses
+import os
+from pathlib import Path
+import subprocess
+from typing import TYPE_CHECKING
+
+import pytest
+
+from code_runner import Example, Limits, RunnerError, run
+from code_runner._examples import parse_examples
+from code_runner._languages import JAVASCRIPT, PYTHON, get_language
+from code_runner._run import Box, _read_result
+from code_runner._sandbox import BoxResult, bwrap_argv, check_sandbox
+from code_runner._starter import parse_starter
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+STARTER = "def f(a):\n"
+EXAMPLE = Example(1, {"a": 1}, 1)
+
+
+@pytest.fixture(autouse=True)
+def _fresh_sandbox_check() -> Iterator[None]:
+    check_sandbox.cache_clear()
+    yield
+    check_sandbox.cache_clear()
+
+
+def _fake(result: BoxResult, files: dict[str, str] | None = None) -> Box:
+    def box(
+        command: list[str],
+        scratch: Path,
+        limits: Limits,
+        memory: int,
+        binds: tuple[str, ...],
+    ) -> BoxResult:
+        for name, text in (files or {}).items():
+            (scratch / name).write_text(text)
+        return result
+
+    return box
+
+
+def _box(**kw: object) -> BoxResult:
+    fields = {
+        "returncode": 0,
+        "timed_out": False,
+        "stdout": "",
+        "stderr": "",
+        "elapsed_ms": 7.0,
+    }
+    return BoxResult(**(fields | kw))
+
+
+def _run(box: Box) -> object:
+    return run("python3", "", STARTER, [EXAMPLE], box=box)[0]
+
+
+def test_crash_without_result_names_the_exit_code() -> None:
+    result = _run(_fake(_box(returncode=139)))
+    assert (result.ok, result.actual, result.error, result.ms) == (
+        False,
+        None,
+        "Crashed (exit 139)",
+        7.0,
+    )
+
+
+def test_out_of_memory_crash_is_named() -> None:
+    stderr = "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory"
+    assert (
+        _run(_fake(_box(returncode=134, stderr=stderr))).error == "MemoryLimitExceeded"
+    )
+
+
+def test_timeout_wins_over_a_result_file() -> None:
+    files = {"result.json": '{"ok": true, "value": 1}'}
+    result = _run(_fake(_box(timed_out=True, elapsed_ms=1000.0), files))
+    assert (result.error, result.ms) == ("TimeLimitExceeded", 1000.0)
+
+
+def test_result_file_variants() -> None:
+    good = _run(
+        _fake(_box(stdout="hi"), {"result.json": '{"ok": true, "value": 1, "ms": 2.5}'})
+    )
+    assert (good.ok, good.ms, good.stdout) == (True, 2.5, "hi")
+    no_ms = _run(_fake(_box(), {"result.json": '{"ok": true, "value": 1}'}))
+    assert no_ms.ms == 7.0
+    failed = _run(_fake(_box(), {"result.json": '{"ok": false}'}))
+    assert failed.error == "Error"
+    named = _run(
+        _fake(_box(), {"result.json": '{"ok": false, "error": "KeyError: x"}'})
+    )
+    assert named.error == "KeyError: x"
+
+
+def test_unreadable_result_files_are_a_crash(tmp_path: Path) -> None:
+    secret = tmp_path / "secret"
+    secret.write_text('{"ok": true, "value": "stolen"}')
+    (tmp_path / "result.json").symlink_to(secret)
+    assert _read_result(tmp_path) is None
+    (tmp_path / "result.json").unlink()
+    os.mkfifo(tmp_path / "result.json")
+    assert _read_result(tmp_path) is None
+    (tmp_path / "result.json").unlink()
+    for text in ("not json", "[1]", '{"no": "ok"}'):
+        (tmp_path / "result.json").write_text(text)
+        assert _read_result(tmp_path) is None
+    assert _read_result(tmp_path / "missing") is None
+
+
+def test_binary_resolution(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    ghost = dataclasses.replace(
+        PYTHON, binary_default="/nope", program="no-such-program-xyz"
+    )
+    monkeypatch.delenv(ghost.binary_env, raising=False)
+    with pytest.raises(RunnerError, match="no-such-program-xyz not found"):
+        ghost.resolve()
+    tool = tmp_path / "bin" / "python3"
+    tool.parent.mkdir()
+    tool.write_text("#!/bin/sh\n")
+    monkeypatch.setenv(ghost.binary_env, str(tool))
+    assert ghost.resolve() == tool
+    assert ghost.ro_binds(tool) == (str(tool.parent.parent),)
+    assert PYTHON.ro_binds(Path("/usr/bin/python3")) == ()
+    assert get_language("TS").name == "typescript"
+
+
+def _has(argv: list[str], seq: list[str]) -> bool:
+    return any(argv[i : i + len(seq)] == seq for i in range(len(argv)))
+
+
+def test_argv_binds_links_and_limits(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    monkeypatch.setattr("code_runner._sandbox._ROOT_LINKS", ("usr", "no-such-root-dir"))
+    argv = bwrap_argv(["prog"], tmp_path, Limits(timeout_s=1.2), 123, ("/opt/x",))
+    assert _has(argv, ["--ro-bind", "/usr", "/usr"])
+    assert _has(argv, ["--ro-bind", "/opt/x", "/opt/x"])
+    assert _has(argv, ["--bind", str(tmp_path), "/work"])
+    assert "--share-net" not in argv
+    assert {"--unshare-all", "--die-with-parent", "--clearenv", "--remount-ro"} <= set(
+        argv
+    )
+    assert "--as=123" in argv
+    assert "--cpu=3" in argv
+    assert argv[-1] == "prog"
+
+
+def test_sandbox_preflight_failures(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr("code_runner._sandbox.PRLIMIT", "/no/such/prlimit")
+    with pytest.raises(RunnerError, match="prlimit"):
+        check_sandbox()
+    monkeypatch.undo()
+
+    def missing(*args: object, **kwargs: object) -> None:
+        message = "bwrap"
+        raise FileNotFoundError(message)
+
+    monkeypatch.setattr(subprocess, "run", missing)
+    with pytest.raises(RunnerError, match="cannot start"):
+        check_sandbox()
+
+    def refused(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]:
+        return subprocess.CompletedProcess(args, 1, b"", b"bwrap: No permissions\n")
+
+    monkeypatch.setattr(subprocess, "run", refused)
+    with pytest.raises(RunnerError, match="No permissions"):
+        check_sandbox()
+
+
+def test_statement_and_starter_edge_cases() -> None:
+    escaped = parse_examples('**Input:** s = "a\\"b,c", t = 1\n**Output:** "x"\n')
+    assert escaped[0].args == {"s": 'a"b,c', "t": 1}
+    assert parse_starter("python3", "def f(a, b").params == ("a", "b")
+    skipped = "class S {\n  if (x) {\n  solve(a) {\n  }\n}\n"
+    assert parse_starter("javascript", skipped).name == "solve"
+
+
+def test_typescript_class_entry_is_exported(tmp_path: Path) -> None:
+    entry = parse_starter(
+        "typescript", "class Solution {\n  solve(a: number): number {\n  }\n}\n"
+    )
+    get_language("typescript").prepare(tmp_path, "class Solution {}", entry, [1])
+    assert "__entry: Solution" in (tmp_path / "solution.cts").read_text()
+    JAVASCRIPT.prepare(tmp_path, "x", entry, [1])
+    assert (tmp_path / "solution.js").read_text() == "x"

code_runner/code_runner/tests/test_isolation.py

@@ -0,0 +1,206 @@
+"""Escape attempts: each one runs hostile code in the real sandbox and must fail."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import socket
+import subprocess
+import time
+from typing import Any, cast
+
+import pytest
+
+from code_runner import CaseResult, Example, Limits, run
+from code_runner.tests.conftest import needs_node, needs_sandbox
+
+pytestmark = needs_sandbox
+STARTER = "def f(a):\n"
+FAST = Limits(timeout_s=1.0)
+
+
+def _one(code: str, lang: str = "python3", limits: Limits = FAST) -> CaseResult:
+    starter = {"python3": STARTER, "javascript": "var f = function(a) {};"}[lang]
+    return run(lang, code, starter, [Example(1, {"a": 0}, 1)], limits=limits)[0]
+
+
+def _pgrep(pattern: str) -> int:
+    done = subprocess.run(
+        ["/usr/bin/pgrep", "-fc", pattern], capture_output=True, text=True, check=False
+    )
+    return int(done.stdout.strip() or 0)
+
+
+def test_no_network_not_even_host_loopback() -> None:
+    with socket.socket() as listener:
+        listener.bind(("127.0.0.1", 0))
+        listener.listen()
+        listener.settimeout(0)
+        port = listener.getsockname()[1]
+        code = (
+            "import socket\n"
+            "def f(a):\n"
+            "    out = []\n"
+            f"    for host, port in [('127.0.0.1', {port}), ('1.1.1.1', 53), ('::1', {port})]:\n"
+            "        try:\n"
+            "            socket.create_connection((host, port), timeout=1).close()\n"
+            "            out.append('connected')\n"
+            "        except OSError:\n"
+            "            out.append('blocked')\n"
+            "    return out\n"
+        )
+        result = _one(code)
+        with pytest.raises(BlockingIOError):
+            listener.accept()
+    assert result.actual == ["blocked", "blocked", "blocked"]
+
+
+@needs_node
+def test_no_network_from_node() -> None:
+    with socket.socket() as listener:
+        listener.bind(("127.0.0.1", 0))
+        listener.listen()
+        port = listener.getsockname()[1]
+        code = (
+            "var f = function(a) {\n"
+            "  const net = process.getBuiltinModule('node:net');\n"
+            "  let seen = 'pending';\n"
+            f"  const s = net.connect({port}, '127.0.0.1');\n"
+            "  s.on('connect', () => { seen = 'connected'; });\n"
+            "  s.on('error', () => {});\n"
+            "  return seen;\n"
+            "};"
+        )
+        result = _one(code, "javascript")
+        listener.settimeout(0.2)
+        with pytest.raises(TimeoutError):
+            listener.accept()
+    assert result.actual == "pending"
+
+
+def test_writes_outside_scratch_never_reach_the_host(tmp_path: Path) -> None:
+    marker = tmp_path / "marker"
+    code = (
+        "def f(a):\n"
+        "    out = []\n"
+        f"    for path in ['/usr/escape', '/etc/escape', '/escape', {str(marker)!r}]:\n"
+        "        try:\n"
+        "            open(path, 'w').write('x')\n"
+        "            out.append('wrote')\n"
+        "        except OSError:\n"
+        "            out.append('denied')\n"
+        "    return out\n"
+    )
+    result = _one(code)
+    assert result.actual == ["denied", "denied", "denied", "denied"]
+    assert not marker.exists()
+    assert not Path("/usr/escape").exists()
+
+
+def test_host_files_and_environment_are_invisible(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    secret = tmp_path / "secret.txt"
+    secret.write_text("hunter2")
+    monkeypatch.setenv("CODE_RUNNER_SECRET", "hunter2")
+    code = (
+        "import os\n"
+        "def f(a):\n"
+        f"    return [os.path.exists({str(secret)!r}), os.path.exists({str(Path.home())!r}),\n"
+        "            'CODE_RUNNER_SECRET' in os.environ, sorted(os.environ), os.getpid() < 20,\n"
+        "            os.path.exists('/proc/1/environ') and 'hunter2' in open('/proc/1/environ').read()]\n"
+    )
+    values = cast("list[Any]", _one(code).actual)
+    exists, home, env_leak, names, own_pidns, proc_leak = values
+    assert (exists, home, env_leak, own_pidns, proc_leak) == (
+        False,
+        False,
+        False,
+        True,
+        False,
+    )
+    assert set(names) <= {"HOME", "LANG", "PATH", "PWD", "PYTHONIOENCODING"}
+
+
+def test_symlink_in_scratch_cannot_exfiltrate_a_host_file(tmp_path: Path) -> None:
+    secret = tmp_path / "secret.txt"
+    secret.write_text("hunter2-secret-contents")
+    code = (
+        "import os\n"
+        "def f(a):\n"
+        f"    os.symlink({str(secret)!r}, '/work/result.json')\n"
+        "    return 7\n"
+    )
+    result = _one(code)
+    assert "hunter2" not in repr(result)
+    assert result.actual == 7  # the harness replaced the link, never followed it
+
+
+def test_fork_bomb_is_contained_and_leaves_nothing_behind() -> None:
+    before = _pgrep("python3 -I /work/harness.py")
+    code = (
+        "import os, subprocess\n"
+        "def f(a):\n"
+        "    subprocess.Popen(['sleep', '97'])\n"
+        "    while True:\n"
+        "        os.fork()\n"
+    )
+    started = time.monotonic()
+    result = _one(code)
+    assert time.monotonic() - started < 6
+    assert result.ok is False
+    assert result.error is not None
+    assert result.error.startswith(("TimeLimitExceeded", "BlockingIOError"))
+    time.sleep(0.3)
+    assert _pgrep("python3 -I /work/harness.py") == before
+    assert _pgrep("^sleep 97$") == 0
+
+
+def test_memory_bomb_is_a_visible_error() -> None:
+    result = _one("def f(a):\n    return len(bytearray(8 * 1024**3))\n")
+    assert result.ok is False
+    assert result.error is not None
+    assert result.error.startswith("MemoryError")
+
+
+@needs_node
+def test_node_heap_bomb_is_a_visible_error() -> None:
+    code = "var f = function(a) { const x = []; for (;;) x.push(new Array(1e6).fill(1)); };"
+    result = _one(code, "javascript", Limits(timeout_s=5.0))
+    assert result.ok is False
+    assert result.error in {"MemoryLimitExceeded", "RangeError: Invalid array length"}
+
+
+def test_output_flood_is_capped() -> None:
+    code = "def f(a):\n    while True:\n        print('x' * 1000)\n"
+    started = time.monotonic()
+    result = _one(code)
+    assert time.monotonic() - started < 6
+    assert result.ok is False
+    assert len(result.stdout) <= 64 * 1024
+
+
+def test_disk_fill_is_capped_by_the_file_size_limit() -> None:
+    code = (
+        "def f(a):\n"
+        "    with open('/work/big', 'wb') as fh:\n"
+        "        while True:\n"
+        "            fh.write(b'x' * 65536)\n"
+    )
+    result = _one(code)
+    assert result.ok is False
+    assert result.error is not None
+    assert "OSError" in result.error
+
+
+def test_timeout_kills_every_process_the_solution_started() -> None:
+    code = (
+        "import subprocess\n"
+        "def f(a):\n"
+        "    subprocess.Popen(['sleep', '97'])\n"
+        "    while True:\n"
+        "        pass\n"
+    )
+    result = _one(code)
+    assert result.error == "TimeLimitExceeded"
+    time.sleep(0.3)
+    assert _pgrep("^sleep 97$") == 0

code_runner/code_runner/tests/test_run.py

@@ -0,0 +1,207 @@
+"""End to end through the real sandbox: every language, every kind of outcome."""
+
+from __future__ import annotations
+
+import pytest
+
+from code_runner import CaseResult, Example, Limits, RunnerError, run
+from code_runner.tests.conftest import (
+    TWO_SUM_CODE,
+    TWO_SUM_STARTER,
+    needs_node,
+    needs_sandbox,
+)
+
+LANGS = [
+    pytest.param("python3", marks=needs_sandbox),
+    pytest.param("javascript", marks=needs_node),
+    pytest.param("typescript", marks=needs_node),
+]
+FAST = Limits(timeout_s=1.0)
+
+
+def _two_sum(lang: str, code: str | None, examples: list[Example]) -> list[CaseResult]:
+    return run(lang, code or TWO_SUM_CODE[lang], TWO_SUM_STARTER[lang], examples)
+
+
+@pytest.mark.parametrize("lang", LANGS)
+def test_correct_solution_passes_every_example(
+    lang: str, two_sum_examples: list[Example]
+) -> None:
+    results = _two_sum(lang, None, two_sum_examples)
+    assert [r.ok for r in results] == [True, True, True]
+    assert [r.n for r in results] == [1, 2, 3]
+    first = results[0]
+    assert first.input == {"nums": [2, 7, 11, 15], "target": 9}
+    assert first.expected == [0, 1]
+    assert sorted(first.actual) == [0, 1]  # any order: [1, 0] is fine
+    assert first.error is None
+    assert first.ms >= 0
+
+
+WRONG = {
+    "python3": "class Solution:\n    def twoSum(self, nums, target):\n        return [0, 0]\n",
+    "javascript": "var twoSum = function(nums, target) { return [0, 0]; };",
+    "typescript": "function twoSum(nums: number[], target: number): number[] { return [0, 0]; }",
+}
+
+
+@pytest.mark.parametrize("lang", LANGS)
+def test_wrong_answer_is_reported_with_actual_value(
+    lang: str, two_sum_examples: list[Example]
+) -> None:
+    results = _two_sum(lang, WRONG[lang], two_sum_examples)
+    assert [r.ok for r in results] == [False, False, False]
+    assert results[0].actual == [0, 0]
+    assert results[0].error is None
+
+
+THROWS = {
+    "python3": "class Solution:\n    def twoSum(self, nums, target):\n        return nums[99]\n",
+    "javascript": "var twoSum = function(nums, target) { throw new RangeError('boom'); };",
+    "typescript": "function twoSum(nums: number[], target: number): number[] { throw new RangeError('boom'); }",
+}
+
+
+@pytest.mark.parametrize("lang", LANGS)
+def test_exception_becomes_an_error_per_case(
+    lang: str, two_sum_examples: list[Example]
+) -> None:
+    results = _two_sum(lang, THROWS[lang], two_sum_examples[:1])
+    only = results[0]
+    assert only.ok is False
+    assert only.actual is None
+    assert only.error is not None
+    assert only.error.split(":")[0] in {"IndexError", "RangeError"}
+
+
+BROKEN = {
+    "python3": "class Solution:\n    def twoSum(self, nums, target) return 1\n",
+    "javascript": "var twoSum = function(nums, target) { return ; ;; }}",
+    "typescript": "function twoSum(nums: number[], target: number): number[] { return [ }",
+}
+
+
+@pytest.mark.parametrize("lang", LANGS)
+def test_syntax_error_is_reported(lang: str, two_sum_examples: list[Example]) -> None:
+    results = _two_sum(lang, BROKEN[lang], two_sum_examples[:2])
+    assert [r.ok for r in results] == [False, False]
+    assert all(r.error and r.error.startswith("SyntaxError") for r in results)
+
+
+PRINTS = {
+    "python3": "class Solution:\n    def twoSum(self, nums, target):\n        print('hi', target)\n        return [0, 1]\n",
+    "javascript": "var twoSum = function(nums, target) { console.log('hi', target); return [0, 1]; };",
+    "typescript": "function twoSum(nums: number[], target: number): number[] { console.log('hi', target); return [0, 1]; }",
+}
+
+
+@pytest.mark.parametrize("lang", LANGS)
+def test_prints_land_in_stdout_per_case(
+    lang: str, two_sum_examples: list[Example]
+) -> None:
+    results = _two_sum(lang, PRINTS[lang], two_sum_examples[:2])
+    assert results[0].stdout.strip() == "hi 9"
+    assert results[1].stdout.strip() == "hi 6"
+
+
+@pytest.mark.parametrize("lang", LANGS)
+def test_missing_entry_point_in_code_is_a_case_error(
+    lang: str, two_sum_examples: list[Example]
+) -> None:
+    code = {
+        "python3": "x = 1\n",
+        "javascript": "var other = 1;",
+        "typescript": "const other = 1;",
+    }
+    results = _two_sum(lang, code[lang], two_sum_examples[:1])
+    assert results[0].ok is False
+    assert results[0].error is not None
+    assert "twoSum" in results[0].error or "Solution" in results[0].error
+
+
+def test_class_style_javascript_and_function_style_python() -> None:
+    js = run(
+        "javascript",
+        "class Solution { solve(a, b) { return a + b; } }",
+        "class Solution {\n  solve(a, b) {\n  }\n}",
+        [Example(1, {"a": 1, "b": 2}, 3)],
+    )
+    py = run(
+        "python3",
+        "def add(a, b):\n    return a + b\n",
+        "def add(a: int, b: int) -> int:\n",
+        [Example(1, {"a": 1, "b": 2}, 3)],
+    )
+    assert js[0].ok is True
+    assert py[0].ok is True
+
+
+def test_python_prelude_and_aliases() -> None:
+    code = "class Solution:\n    def f(self, xs: List[int]) -> int:\n        return collections.Counter(xs)[1]\n"
+    starter = "class Solution:\n    def f(self, xs: List[int]) -> int:\n"
+    out = run("python", code, starter, [Example(1, {"xs": [1, 1, 2]}, 2)])
+    assert out[0].ok is True
+
+
+def test_positional_fallback_when_statement_names_differ() -> None:
+    out = run(
+        "python3",
+        "def add(a, b):\n    return a - b\n",
+        "def add(a, b):\n",
+        [Example(1, {"x": 5, "y": 2}, 3)],
+    )
+    assert out[0].ok is True
+
+
+def test_argument_count_mismatch_is_a_case_error_without_running() -> None:
+    out = run(
+        "python3",
+        "def add(a, b):\n    return a\n",
+        "def add(a, b):\n",
+        [Example(1, {"only": 5}, 3)],
+    )
+    assert out[0].ok is False
+    assert out[0].error is not None
+    assert out[0].error.startswith("ArgumentError")
+
+
+def test_non_json_result_is_a_case_error() -> None:
+    out = run(
+        "python3",
+        "def f(a):\n    return {1, 2}\n",
+        "def f(a):\n",
+        [Example(1, {"a": 1}, [1, 2])],
+    )
+    assert out[0].error is not None
+    assert out[0].error.startswith("TypeError")
+
+
+def test_deep_order_and_float_tolerance() -> None:
+    out = run(
+        "python3",
+        "def f(a):\n    return [[2, 1], [0, -1]]\n",
+        "def f(a):\n",
+        [Example(1, {"a": 0}, [[-1, 0], [1, 2]], "deep")],
+    )
+    assert out[0].ok is True
+
+
+def test_timeout_is_per_case() -> None:
+    out = run(
+        "python3",
+        "def f(a):\n    while a:\n        pass\n    return 1\n",
+        "def f(a):\n",
+        [Example(1, {"a": 1}, 1), Example(2, {"a": 0}, 1)],
+        limits=FAST,
+    )
+    assert out[0].error == "TimeLimitExceeded"
+    assert out[0].ok is False
+    assert out[0].ms >= 900
+    assert out[1].ok is True
+
+
+def test_unsupported_language_and_empty_examples() -> None:
+    with pytest.raises(RunnerError, match="language"):
+        run("cobol", "", "", [])
+    assert run("python3", "def f(a): pass\n", "def f(a):\n", []) == []

code_runner/code_runner/tests/test_starter.py

@@ -0,0 +1,78 @@
+import pytest
+
+from code_runner._model import RunnerError
+from code_runner._starter import Entry, extract_starter, parse_starter
+
+PY_CLASS = """class Solution:
+    def twoSum(self, nums: list[int], target: int) -> list[int]:
+"""
+PY_COMMENTED = """# Definition for singly-linked list.
+# class ListNode:
+#     def __init__(self, val=0, next=None):
+class Solution:
+    def solve(self, grid: Dict[str, Tuple[int, int]], k: int = 3, *rest) -> int:
+        pass
+"""
+PY_FUNC = "def add(a: int, b: int = 2) -> int:\n    pass\n"
+JS_VAR = """/**
+ * @param {number[]} nums
+ */
+var twoSum = function(nums, target) {
+
+};"""
+JS_FUNC = "function isPalindrome(x, y = 3) {\n}\n"
+JS_ARROW = "const f = (a, b) => {\n}\n"
+JS_CLASS = "class Solution {\n  constructor() {}\n  solve(nums, ...rest) {\n  }\n}\n"
+TS_FUNC = "function twoSum(nums: number[], target: number): number[] {\n};"
+TS_ARROW = "const f = (a: Map<string, number>, b?: number): number => {\n}\n"
+
+
+@pytest.mark.parametrize(
+    ("lang", "text", "want"),
+    [
+        ("python3", PY_CLASS, Entry("Solution", "twoSum", ("nums", "target"))),
+        ("python3", PY_COMMENTED, Entry("Solution", "solve", ("grid", "k"))),
+        ("python3", PY_FUNC, Entry(None, "add", ("a", "b"))),
+        ("javascript", JS_VAR, Entry(None, "twoSum", ("nums", "target"))),
+        ("javascript", JS_FUNC, Entry(None, "isPalindrome", ("x", "y"))),
+        ("javascript", JS_ARROW, Entry(None, "f", ("a", "b"))),
+        ("javascript", JS_CLASS, Entry("Solution", "solve", ("nums",))),
+        ("typescript", TS_FUNC, Entry(None, "twoSum", ("nums", "target"))),
+        ("typescript", TS_ARROW, Entry(None, "f", ("a", "b"))),
+    ],
+)
+def test_parse_starter(lang: str, text: str, want: Entry) -> None:
+    assert parse_starter(lang, text) == want
+
+
+@pytest.mark.parametrize(
+    ("lang", "text"),
+    [
+        ("python3", "x = 1\n"),
+        ("python3", "class Solution:\n    pass\n"),
+        ("javascript", "// nothing here\n"),
+        ("javascript", "class Solution {\n  constructor() {}\n}\n"),
+        ("typescript", ""),
+    ],
+)
+def test_starter_without_entry_is_refused(lang: str, text: str) -> None:
+    with pytest.raises(RunnerError, match="entry point"):
+        parse_starter(lang, text)
+
+
+def test_extract_starter_takes_the_fenced_block() -> None:
+    statement = (
+        "# 1. Two Sum (Easy)\n\nbody\n\n## Starter code (python3)\n```\n"
+        + PY_CLASS
+        + "```\n\n## Site hints\nnope\n"
+    )
+    assert extract_starter(statement) == PY_CLASS
+
+
+def test_extract_starter_without_section_is_empty() -> None:
+    assert extract_starter("# 1. Two Sum\n\nno starter\n") == ""
+
+
+def test_extract_starter_unfenced_runs_to_next_section() -> None:
+    text = "intro\n## Starter code (javascript)\nvar f = function(a) {};\n## Hints\nx\n"
+    assert extract_starter(text) == "var f = function(a) {};\n"

code_runner/install.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Editable user install so /usr/bin/python3 (the tutor server, the CLI) can
+# `import code_runner`. Stdlib only; idempotent.
+set -euo pipefail
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly REPO_DIR
+readonly SYSTEM_PYTHON="/usr/bin/python3"
+
+"$SYSTEM_PYTHON" -m pip install --user --no-deps --break-system-packages -e "$REPO_DIR"
+cd /
+"$SYSTEM_PYTHON" -c 'import code_runner; print(code_runner.__file__)'

code_runner/pyproject.toml

@@ -0,0 +1,198 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "code_runner"
+version = "0.1.0"
+description = "Run a learner solution (Python, JavaScript, TypeScript) against a problem's examples in a bubblewrap sandbox"
+requires-python = ">=3.14"
+# Stdlib only on purpose: the tutor server imports it from the system python.
+dependencies = []
+
+[tool.setuptools.packages.find]
+include = ["code_runner*"]
+
+[tool.setuptools.package-data]
+code_runner = ["py.typed", "harness/*"]
+
+[tool.ruff]
+target-version = "py314"
+include = ["*.py", "**/*.py"]
+exclude = [".git", ".venv", "__pycache__", "build", "dist", ".eggs"]
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "CPY001",
+    "D100",
+    "D101",
+    "D102",
+    "D103",
+    "D104",
+    "D105",
+    "D107",
+    "D203",
+    "D213",
+    "COM812",
+    "ISC001",
+    "S603",
+]
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.lint.per-file-ignores]
+"scripts/*.py" = ["INP001"]
+# The sandbox-side harness is a program of its own: it reads its job from files.
+# The sandbox mounts its own private tmpfs at /tmp.
+"**/_sandbox.py" = ["S108"]
+"**/harness/*.py" = ["INP001", "T201", "S102", "BLE001", "PTH123"]
+# Tests embed the learner programs they run as source strings.
+"**/tests/**/*.py" = ["ARG", "D", "E501", "PLC0415", "PLR2004", "S101", "SLF001"]
+"**/test_*.py" = ["ARG", "D", "E501", "PLC0415", "PLR2004", "S101", "SLF001"]
+
+[tool.ruff.lint.pylint]
+max-args = 6
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.isort]
+force-single-line = false
+force-sort-within-sections = true
+known-first-party = ["code_runner"]
+
+[tool.ruff.lint.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "double"
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+docstring-code-format = true
+
+[tool.mypy]
+python_version = "3.14"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+disallow_any_unimported = true
+disallow_any_explicit = false
+disallow_any_generics = true
+disallow_subclassing_any = true
+strict_equality = true
+extra_checks = true
+ignore_missing_imports = true
+show_error_codes = true
+color_output = true
+exclude = [".venv/"]
+
+[tool.pylint.main]
+# The repo root on the path, so pylint resolves the package's own imports the
+# same way pytest does instead of reporting import-error for them.
+init-hook = "import sys; sys.path.insert(0, 'code_runner')"
+analyse-fallback-blocks = true
+persistent = true
+jobs = 0
+py-version = "3.14"
+ignore = [".venv", "__pycache__"]
+# Test suites intentionally use patterns (protected-access, fixture-arg
+# shadowing) that don't apply to source code; ruff's per-file-ignores carry
+# the equivalent exemptions, so tests stay linted -- by ruff. This is the same
+# source-only pylint scope wake-alarm / diet-guard / testsAndMisc use. It has
+# to be `ignore-paths`, not `ignore = ["tests"]`: `ignore` only matches the
+# basename of what pylint is *given*, and pre-commit hands it explicit file
+# paths, so a directory name in `ignore` never fires under the hook.
+# harness/ runs inside the sandbox and must exec the solution it is given.
+ignore-paths = ["^(.*/)?tests/.*$", "^(.*/)?harness/.*$"]
+ignore-patterns = [".*\\.pyi$"]
+
+[tool.pylint.messages_control]
+enable = "all"
+disable = [
+    "missing-module-docstring",
+    "missing-class-docstring",
+    "missing-function-docstring",
+    "use-implicit-booleaness-not-comparison-to-zero",
+]
+
+[tool.pylint.design]
+max-args = 6
+min-public-methods = 0
+max-module-lines = 1000
+max-attributes = 10
+
+[tool.pylint.typecheck]
+generated-members = [
+    ".*\\.assert_called_once_with",
+    ".*\\.assert_called_once",
+    ".*\\.assert_called",
+    ".*\\.assert_not_called",
+    ".*\\.assert_any_call",
+    ".*\\.call_args",
+    ".*\\.call_args_list",
+    ".*\\.call_count",
+]
+
+[tool.bandit]
+exclude_dirs = ["tests", ".venv"]
+
+[tool.pytest.ini_options]
+markers = ["sandbox: needs bubblewrap with unprivileged user namespaces (skipped otherwise)"]
+testpaths = ["code_runner/tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--strict-markers",
+    "--strict-config",
+    "-ra",
+    "--cov=code_runner",
+    "--cov-branch",
+    "--cov-report=term-missing",
+    "--cov-report=lcov",
+]
+filterwarnings = [
+    "error",
+    "ignore::DeprecationWarning",
+    "default::pytest.PytestUnraisableExceptionWarning",
+]
+
+[tool.coverage.run]
+source = ["code_runner"]
+branch = true
+omit = [
+    # Runs inside the sandbox in another process; behaviour is tested through run().
+    "*/harness/*",
+    "*/__pycache__/*",
+    "*/tests/*",
+    "*/.venv/*",
+]
+
+[tool.coverage.report]
+fail_under = 100
+show_missing = true
+skip_covered = false
+exclude_lines = [
+    "pragma: no cover",
+    "raise NotImplementedError",
+    "raise AssertionError",
+    "if TYPE_CHECKING:",
+    'if __name__ == "__main__":',
+]
+partial_branches = ["pragma: no branch"]

code_runner/requirements.txt

@@ -0,0 +1,10 @@
+# Exact pins, enforced by the shared dependency-freshness gate: a `>=`
+# floor is satisfied by anything newer, so it tells a checker nothing
+# about whether this repo is actually current.
+bandit[toml]==1.9.4
+mypy==2.4.1
+pre-commit==4.7.0
+pylint==4.1.3
+pytest==9.2.0
+pytest-cov==7.1.0
+ruff==0.17.3