From fe4e34287abd9318d09fddb5cd7e2eeef70f0d19 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Tue, 10 Nov 2026 14:14:58 +0100 Subject: [PATCH] Move code_runner to the tutor repo It has one consumer, kuhyx/tutor (packages/code_runner). Tags code-runner-v0.1.0 and -v0.1.1 stay so old pins still resolve. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WzxdWYFhDShgeHByh6yT9L --- .github/workflows/code-runner-precommit.yml | 73 ------ .github/workflows/code-runner-tests.yml | 34 --- code_runner/.pre-commit-config.yaml | 143 ----------- code_runner/README.md | 12 - code_runner/code_runner/__init__.py | 20 -- code_runner/code_runner/_compare.py | 89 ------- code_runner/code_runner/_examples.py | 75 ------ code_runner/code_runner/_languages.py | 119 --------- code_runner/code_runner/_model.py | 40 --- code_runner/code_runner/_run.py | 134 ---------- code_runner/code_runner/_sandbox.py | 147 ----------- code_runner/code_runner/_split.py | 29 --- code_runner/code_runner/_starter.py | 108 -------- code_runner/code_runner/harness/harness.js | 45 ---- code_runner/code_runner/harness/harness.py | 69 ----- code_runner/code_runner/py.typed | 0 code_runner/code_runner/tests/__init__.py | 0 code_runner/code_runner/tests/conftest.py | 70 ------ code_runner/code_runner/tests/test_compare.py | 65 ----- .../code_runner/tests/test_examples.py | 106 -------- .../code_runner/tests/test_internals.py | 235 ------------------ .../code_runner/tests/test_isolation.py | 206 --------------- code_runner/code_runner/tests/test_run.py | 227 ----------------- code_runner/code_runner/tests/test_starter.py | 78 ------ code_runner/install.sh | 12 - code_runner/pyproject.toml | 198 --------------- code_runner/requirements.txt | 10 - 27 files changed, 2344 deletions(-) delete mode 100644 .github/workflows/code-runner-precommit.yml delete mode 100644 .github/workflows/code-runner-tests.yml delete mode 100644 code_runner/.pre-commit-config.yaml delete mode 100644 code_runner/README.md delete mode 100644 code_runner/code_runner/__init__.py delete mode 100644 code_runner/code_runner/_compare.py delete mode 100644 code_runner/code_runner/_examples.py delete mode 100644 code_runner/code_runner/_languages.py delete mode 100644 code_runner/code_runner/_model.py delete mode 100644 code_runner/code_runner/_run.py delete mode 100644 code_runner/code_runner/_sandbox.py delete mode 100644 code_runner/code_runner/_split.py delete mode 100644 code_runner/code_runner/_starter.py delete mode 100644 code_runner/code_runner/harness/harness.js delete mode 100644 code_runner/code_runner/harness/harness.py delete mode 100644 code_runner/code_runner/py.typed delete mode 100644 code_runner/code_runner/tests/__init__.py delete mode 100644 code_runner/code_runner/tests/conftest.py delete mode 100644 code_runner/code_runner/tests/test_compare.py delete mode 100644 code_runner/code_runner/tests/test_examples.py delete mode 100644 code_runner/code_runner/tests/test_internals.py delete mode 100644 code_runner/code_runner/tests/test_isolation.py delete mode 100644 code_runner/code_runner/tests/test_run.py delete mode 100644 code_runner/code_runner/tests/test_starter.py delete mode 100755 code_runner/install.sh delete mode 100644 code_runner/pyproject.toml delete mode 100644 code_runner/requirements.txt diff --git a/.github/workflows/code-runner-precommit.yml b/.github/workflows/code-runner-precommit.yml deleted file mode 100644 index f427d98..0000000 --- a/.github/workflows/code-runner-precommit.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: code-runner pre-commit - -on: - push: - branches: [main] - # The workflow's own file counts as code_runner: editing only this file would - # otherwise trigger nothing, so a fix to the job itself could never prove - # itself. - paths: ['code_runner/**', '.github/workflows/code-runner-precommit.yml'] - pull_request: - branches: [main] - paths: ['code_runner/**', '.github/workflows/code-runner-precommit.yml'] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - # Only pre-commit itself. Every linting hook runs in its own isolated - # env -- ruff, mypy, pylint, bandit and codespell each come from their - # own repo+rev, and the local hooks need nothing beyond bash and - # python3 -- so installing the full requirements.txt spent 9s a run on - # packages no pre-commit-stage hook imports. Verified by running all - # three configs green in a venv holding pre-commit and nothing else. - # The pin is read out of requirements.txt rather than repeated here, so - # it cannot drift from what the dependency-freshness gate governs. - - name: Install pre-commit - run: | - pip install "$(grep '^pre-commit==' code_runner/requirements.txt)" - - # Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on - # hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three - # package jobs shared one key, raced to overwrite it with hook sets built - # from four different configs, and editing a per-package config did not - # invalidate it. The cache was therefore silently wrong rather than - # missing, and the hooks reinstalled from scratch on every run (38-49s). - - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - key: >- - pre-commit-code-runner-${{ env.pythonLocation }}-${{ - hashFiles('code_runner/.pre-commit-config.yaml') }} - # Prefix fallback, or this fix would evaporate on the very - # event that motivated it: bumping a hook rev changes the - # exact key, and a total miss means reinstalling every hook - # again. pre-commit's store is keyed internally by hook - # repo+rev, so a superseded entry is ignored disk rather - # than a wrong hook, and cache still re-saves under the new - # exact key after a prefix-only hit. - restore-keys: pre-commit-code-runner-${{ env.pythonLocation }}- - # mypy re-analyses the stdlib and every third-party stub from scratch - # without this, which measured 10s of the hook step in CI against 0.25s - # warm locally -- a 19MB cache dir does the whole difference. Keyed on - # the config, not the sources: mypy's own cache metadata records the - # interpreter, the flag set and a hash per source file, so it re-checks - # what changed and cannot pass stale code. Deliberately NOT keyed on - # github.sha -- that re-saves 19MB x3 every commit and would evict the - # 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps - # the expensive part, the dependency analysis, which is what costs 10s. - - uses: actions/cache@v4 - with: - path: .mypy_cache - key: >- - mypy-code-runner-${{ env.pythonLocation }}-${{ - hashFiles('code_runner/.pre-commit-config.yaml') }} - restore-keys: mypy-code-runner-${{ env.pythonLocation }}- - - name: Run the hooks - run: >- - pre-commit run --config code_runner/.pre-commit-config.yaml - --all-files --show-diff-on-failure diff --git a/.github/workflows/code-runner-tests.yml b/.github/workflows/code-runner-tests.yml deleted file mode 100644 index f21688d..0000000 --- a/.github/workflows/code-runner-tests.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: code-runner tests - -on: - push: - branches: [main] - # The workflow's own file counts as code_runner: editing only this file would - # otherwise trigger nothing, so a fix to the job itself could never prove - # itself. - paths: ['code_runner/**', '.github/workflows/code-runner-tests.yml'] - pull_request: - branches: [main] - paths: ['code_runner/**', '.github/workflows/code-runner-tests.yml'] - -jobs: - test: - runs-on: ubuntu-latest - defaults: - run: - working-directory: code_runner - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - cache: pip - cache-dependency-path: code_runner/requirements.txt - - - name: Install dependencies - run: pip install -r requirements.txt && pip install --no-deps -e . - - - name: Run tests with coverage - run: python -m pytest -v diff --git a/code_runner/.pre-commit-config.yaml b/code_runner/.pre-commit-config.yaml deleted file mode 100644 index 08be4d5..0000000 --- a/code_runner/.pre-commit-config.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# Pre-commit Configuration for code_runner -# Install: pre-commit install && pre-commit install --hook-type pre-push -# Run: pre-commit run --all-files -# Update: pre-commit autoupdate - -# NOTE: this subproject lives inside the ~/src/utils monorepo, whose git root is -# one level up. pre-commit always runs hooks from the git root, so every path -# below is written relative to THAT, not to this directory, and the top-level -# `files` key keeps these hooks off the sibling subprojects. CI invokes this -# config exactly the same way: -# pre-commit run --config code_runner/.pre-commit-config.yaml --all-files -files: ^code_runner/ - -default_language_version: - python: python3 -default_stages: [pre-commit] -fail_fast: false - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 - hooks: - - id: trailing-whitespace - args: [--markdown-linebreak-ext=md] - - id: end-of-file-fixer - - id: check-yaml - - id: check-json - - id: check-toml - - id: check-added-large-files - args: [--maxkb=2000] - - id: check-merge-conflict - - id: detect-private-key - - id: debug-statements - - id: name-tests-test - args: [--pytest-test-first] - - id: check-ast - - id: mixed-line-ending - args: [--fix=lf] - - id: requirements-txt-fixer - - - repo: local - hooks: - - id: file-length-cap - name: file length <= 250 lines - entry: bash scripts/check_file_length.sh - language: system - pass_filenames: true - - id: subproject-scope - name: subproject scope regex still matches - entry: scripts/check_subproject_scope.sh code_runner - language: script - # always_run so the top-level `files` filter cannot skip the one hook - # whose job is to notice that filter has stopped matching anything. - always_run: true - pass_filenames: false - - id: no-noqa - name: Block noqa comments - entry: '(?i)#\s*(noqa|type:\s*ignore)' - language: pygrep - types: [python] - - id: no-ruff-noqa - name: Block ruff noqa file-level comments - entry: '(?i)#\s*ruff:\s*noqa' - language: pygrep - types: [python] - - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.17.2 - hooks: - - id: ruff - args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix, --show-fixes] - types_or: [python, pyi] - - id: ruff-format - types_or: [python, pyi] - - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.13.0 - hooks: - - id: mypy - args: - - --ignore-missing-imports - - --no-error-summary - - --disable-error-code=no-untyped-def - - --disable-error-code=no-untyped-call - - --disable-error-code=var-annotated - - --disable-error-code=no-any-unimported - - --disable-error-code=type-arg - - --disable-error-code=no-any-return - - --disable-error-code=misc - - --disable-error-code=unused-ignore - - --disable-error-code=unreachable - - --disable-error-code=assignment - - --disable-error-code=no-redef - - --disable-error-code=attr-defined - - --disable-error-code=arg-type - - --disable-error-code=union-attr - - --disable-error-code=call-overload - - --disable-error-code=return-value - - --disable-error-code=redundant-cast - - --disable-error-code=empty-body - - --disable-error-code=list-item - - - repo: https://github.com/pylint-dev/pylint - rev: v4.0.8 - hooks: - - id: pylint - args: - - --rcfile=code_runner/pyproject.toml - - --fail-under=10 - - --jobs=4 - additional_dependencies: - - pytest - - - repo: https://github.com/PyCQA/bandit - rev: 1.9.4 - hooks: - - id: bandit - args: - - -c - - code_runner/pyproject.toml - - --severity-level=high - - --confidence-level=medium - - --skip=B113 - additional_dependencies: ["bandit[toml]"] - exclude: ^(code_runner/code_runner/tests/|.*test.*\.py$) - - - repo: local - hooks: - - id: pytest-coverage - name: pytest with coverage enforcement - entry: scripts/run_subproject_tests.sh code_runner - language: script - types: [python] - pass_filenames: false - require_serial: true - stages: [pre-push] - - - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 - hooks: - - id: codespell - args: - - --skip=*.json,*.lock,.git,__pycache__,.venv diff --git a/code_runner/README.md b/code_runner/README.md deleted file mode 100644 index 86145a3..0000000 --- a/code_runner/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# code_runner - -Runs a learner's solution (Python 3, JavaScript, TypeScript) against a problem statement's examples inside bubblewrap: no network, read-only `/usr`, scratch dir, CPU/memory/process/file/time limits. - -```python -import code_runner as cr - -statement = ... # markdown from ~/.claude/skills/leetcode/fetch_problem.py -results = cr.run("python3", code, cr.extract_starter(statement), cr.parse_examples(statement)) -``` - -`run(lang, code, starter, examples) -> list[CaseResult]` with `n, input, expected, actual, ok, stdout, error, ms`. One sandbox process per example. TypeScript runs through node's type stripping (no type check, no enums). Needs `bwrap`, `prlimit`, `/usr/bin/python3`, node >= 22.18. Install: `./install.sh`. diff --git a/code_runner/code_runner/__init__.py b/code_runner/code_runner/__init__.py deleted file mode 100644 index 5dfd6dc..0000000 --- a/code_runner/code_runner/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Run a learner's solution against a problem's examples in a bubblewrap sandbox.""" - -from __future__ import annotations - -from code_runner._examples import parse_examples -from code_runner._model import CaseResult, Example, Limits, RunnerError -from code_runner._run import run -from code_runner._starter import Entry, extract_starter, parse_starter - -__all__ = [ - "CaseResult", - "Entry", - "Example", - "Limits", - "RunnerError", - "extract_starter", - "parse_examples", - "parse_starter", - "run", -] diff --git a/code_runner/code_runner/_compare.py b/code_runner/code_runner/_compare.py deleted file mode 100644 index 6bb6c45..0000000 --- a/code_runner/code_runner/_compare.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Output comparison: exact JSON equality, float tolerance, order rules.""" - -from __future__ import annotations - -import json -import math - -TOLERANCE = 1e-5 # LeetCode accepts floating answers within 1e-5 -MAX_DEPTH = 64 # real answers nest a handful of levels; deeper is hostile - - -class TooDeepError(ValueError): - """A compared value nests deeper than ``MAX_DEPTH``.""" - - -def too_deep(value: object, limit: int = MAX_DEPTH) -> bool: - """Whether ``value`` nests lists/dicts deeper than ``limit`` (iterative).""" - stack: list[tuple[object, int]] = [(value, 0)] - while stack: - node, depth = stack.pop() - if isinstance(node, list | dict): - if depth >= limit: - return True - children = node if isinstance(node, list) else node.values() - stack.extend((child, depth + 1) for child in children) - return False - - -def _as_number(value: object) -> float | None: - if isinstance(value, bool) or not isinstance(value, int | float): - return None - return value - - -def _key(value: object) -> str: - return json.dumps(value, sort_keys=True, ensure_ascii=False) - - -def _deep_sorted(value: object) -> object: - if isinstance(value, list): - return sorted((_deep_sorted(item) for item in value), key=_key) - return value - - -def _equal(expected: object, actual: object) -> bool: - want, got = _as_number(expected), _as_number(actual) - if want is not None and got is not None: - return math.isclose(want, got, rel_tol=TOLERANCE, abs_tol=TOLERANCE) - if isinstance(expected, list) and isinstance(actual, list): - return len(expected) == len(actual) and all( - _equal(e, a) for e, a in zip(expected, actual, strict=True) - ) - if isinstance(expected, dict) and isinstance(actual, dict): - return expected.keys() == actual.keys() and all( - _equal(expected[k], actual[k]) for k in expected - ) - return type(expected) is type(actual) and expected == actual - - -def _top_sorted(value: object) -> object: - if isinstance(value, list): - return sorted(value, key=_key) - return value - - -def matches(expected: object, actual: object, order: str) -> bool: - """Whether ``actual`` is an accepted answer for ``expected``. - - Args: - expected: the statement's answer (JSON value). - actual: what the solution returned (JSON value). - order: "exact", "any" (list order free) or "deep" (nested order free too). - - Raises: - ValueError: ``order`` is none of those. - TooDeepError: a value nests deeper than ``MAX_DEPTH`` (the recursive - walks below would blow the stack). - """ - if too_deep(expected) or too_deep(actual): - msg = f"value nested deeper than {MAX_DEPTH} levels" - raise TooDeepError(msg) - if order == "exact": - return _equal(expected, actual) - if order == "any": - return _equal(_top_sorted(expected), _top_sorted(actual)) - if order == "deep": - return _equal(_deep_sorted(expected), _deep_sorted(actual)) - msg = f"unknown order {order!r}" - raise ValueError(msg) diff --git a/code_runner/code_runner/_examples.py b/code_runner/code_runner/_examples.py deleted file mode 100644 index 9e90209..0000000 --- a/code_runner/code_runner/_examples.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -import ast -import json -import logging -import re -from typing import Final - -from code_runner._model import Example, Order -from code_runner._split import split_top_level - -_logger: Final = logging.getLogger(__name__) -_LABEL: Final = re.compile(r"^(Input|Output)\s*:\s*(.*)$") -_PAIR: Final = re.compile(r"^([A-Za-z_]\w*)\s*=\s*(.+)$", re.DOTALL) -_ANY_ORDER: Final = re.compile(r"any order|order of the output", re.IGNORECASE) -_DEEP_ORDER: Final = re.compile( - r"order of the (?:triplets|quadruplets|subsets|groups|pairs|elements)", - re.IGNORECASE, -) - - -def _value(text: str) -> object: - try: - return json.loads(text) - except ValueError: - try: - return ast.literal_eval(text) - except (ValueError, SyntaxError) as exc: - raise ValueError(text[:40]) from exc - - -def _arguments(text: str) -> dict[str, object]: - args: dict[str, object] = {} - for part in split_top_level(text): - pair = _PAIR.match(part) - if pair is None: - msg = f"not a name = value pair: {part[:40]!r}" - raise ValueError(msg) - args[pair.group(1)] = _value(pair.group(2)) - return args - - -def _labels(body: str) -> list[tuple[str, str]]: - cleaned = ( - line.replace("**", "").replace("`", "").strip() for line in body.splitlines() - ) - matched = (_LABEL.match(line) for line in cleaned) - return [(m.group(1), m.group(2).strip()) for m in matched if m is not None] - - -def _order(body: str) -> Order: - if _DEEP_ORDER.search(body): - return "deep" - return "any" if _ANY_ORDER.search(body) else "exact" - - -def parse_examples(statement: str) -> list[Example]: - body = re.split(r"^## ", statement, maxsplit=1, flags=re.MULTILINE)[0] - order = _order(body) - found: list[Example] = [] - pending: str | None = None - number = 0 - for label, text in _labels(body): - if label == "Input": - pending = text - continue - if pending is None: - continue - number += 1 - try: - found.append(Example(number, _arguments(pending), _value(text), order)) - except ValueError as exc: - _logger.warning("example %d skipped, not runnable: %s", number, exc) - pending = None - return found diff --git a/code_runner/code_runner/_languages.py b/code_runner/code_runner/_languages.py deleted file mode 100644 index 36be09d..0000000 --- a/code_runner/code_runner/_languages.py +++ /dev/null @@ -1,119 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from importlib import resources -import json -import os -from pathlib import Path -import shutil -from typing import TYPE_CHECKING - -from code_runner._model import RunnerError -from code_runner._sandbox import WORK - -if TYPE_CHECKING: - from code_runner._starter import Entry - -MEGABYTE = 1024 * 1024 -_NODE_FLAGS = ( - "--permission", - f"--allow-fs-read={WORK}", - f"--allow-fs-write={WORK}", - "--max-old-space-size=128", -) - - -@dataclass(frozen=True) -class Language: - name: str - solution_file: str - harness_file: str - program: str - binary_env: str - binary_default: str - flags: tuple[str, ...] - memory_bytes: int - - def resolve(self) -> Path: - for candidate in ( - os.environ.get(self.binary_env), - self.binary_default, - shutil.which(self.program), - ): - if candidate and Path(candidate).is_file(): - return Path(candidate).resolve() - msg = f"{self.program} not found (set {self.binary_env})" - raise RunnerError(msg) - - def command(self, binary: Path) -> list[str]: - return [str(binary), *self.flags, f"{WORK}/{self.harness_file}", WORK] - - def ro_binds(self, binary: Path) -> tuple[str, ...]: - if binary.is_relative_to("/usr"): - return () - return (str(binary.parent.parent),) - - def prepare( - self, scratch: Path, code: str, entry: Entry, args: list[object] - ) -> None: - harness = resources.files("code_runner").joinpath("harness", self.harness_file) - (scratch / self.harness_file).write_text(harness.read_text(encoding="utf-8")) - text = code - if self.name == "typescript": - text += f"\nmodule.exports = {{ __entry: {entry.cls or entry.name} }};\n" - (scratch / self.solution_file).write_text(text, encoding="utf-8") - spec = { - "cls": entry.cls, - "fn": entry.name, - "args": args, - "typescript": self.name == "typescript", - } - (scratch / "input.json").write_text(json.dumps(spec), encoding="utf-8") - - -PYTHON = Language( - "python3", - "solution.py", - "harness.py", - "python3", - "CODE_RUNNER_PYTHON", - "/usr/bin/python3", - ("-I", "-B"), - 512 * MEGABYTE, -) -JAVASCRIPT = Language( - "javascript", - "solution.js", - "harness.js", - "node", - "CODE_RUNNER_NODE", - "/usr/bin/node", - _NODE_FLAGS, - 2048 * MEGABYTE, -) -TYPESCRIPT = Language( - "typescript", - "solution.cts", - "harness.js", - "node", - "CODE_RUNNER_NODE", - "/usr/bin/node", - _NODE_FLAGS, - 2048 * MEGABYTE, -) -_BY_NAME = { - "python3": PYTHON, - "python": PYTHON, - "javascript": JAVASCRIPT, - "js": JAVASCRIPT, - "typescript": TYPESCRIPT, - "ts": TYPESCRIPT, -} - - -def get_language(name: str) -> Language: - try: - return _BY_NAME[name.lower()] - except KeyError: - msg = f"unsupported language {name!r}; use python3, javascript or typescript" - raise RunnerError(msg) from None diff --git a/code_runner/code_runner/_model.py b/code_runner/code_runner/_model.py deleted file mode 100644 index dbcad4e..0000000 --- a/code_runner/code_runner/_model.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -# "any": list order is free; "deep": the order inside its items is free too. -Order = Literal["exact", "any", "deep"] -MAX_CAPTURE = 64 * 1024 - - -class RunnerError(Exception): - pass - - -@dataclass(frozen=True) -class Example: - n: int - args: dict[str, object] - expected: object - order: Order = "exact" - - -@dataclass(frozen=True) -class CaseResult: - n: int - input: dict[str, object] - expected: object - actual: object - ok: bool - stdout: str = "" - error: str | None = None - ms: float = 0.0 - - -@dataclass(frozen=True) -class Limits: - timeout_s: float = 2.0 - nproc: int = 64 - fsize_bytes: int = 1024 * 1024 - nofile: int = 256 diff --git a/code_runner/code_runner/_run.py b/code_runner/code_runner/_run.py deleted file mode 100644 index 3d668e1..0000000 --- a/code_runner/code_runner/_run.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Sequence -from dataclasses import dataclass -import json -import os -from pathlib import Path -import stat -import tempfile - -from code_runner._compare import TooDeepError, matches, too_deep -from code_runner._languages import Language, get_language -from code_runner._model import MAX_CAPTURE, CaseResult, Example, Limits -from code_runner._sandbox import BoxResult, execute -from code_runner._starter import Entry, parse_starter - -Box = Callable[[list[str], Path, Limits, int, tuple[str, ...]], BoxResult] -_OUT_OF_MEMORY = ("out of memory", "allocation failed") -_MAX_ERROR = 200 -# Everything hostile JSON can raise on the host: deep nesting, huge numbers. -_HOSTILE = (ValueError, TypeError, RecursionError, MemoryError, OverflowError) - - -def _positional(example: Example, entry: Entry) -> list[object]: - if set(example.args) == set(entry.params): - return [example.args[name] for name in entry.params] - if len(example.args) == len(entry.params): - return list(example.args.values()) - given, wanted = len(example.args), len(entry.params) - msg = f"ArgumentError: example gives {given} arguments, {entry.name} takes {wanted}" - raise ValueError(msg) - - -def _read_result(scratch: Path) -> dict[str, object] | None: - # The scratch dir is the solution's: never follow a link it planted there. - flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK - try: - fd = os.open(scratch / "result.json", flags) - except OSError: - return None - with os.fdopen(fd, "rb") as handle: - if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): - return None - raw = handle.read(MAX_CAPTURE * 16) - try: - payload = json.loads(raw) - except _HOSTILE: # ValueError, or RecursionError on "[" * 200000 - return None - if not isinstance(payload, dict) or "ok" not in payload: - return None - return None if too_deep(payload) else payload - - -def _crash(box: BoxResult) -> str: - low = box.stderr.lower() - if any(marker in low for marker in _OUT_OF_MEMORY): - return "MemoryLimitExceeded" - return f"Crashed (exit {box.returncode})" - - -def _took(ms: object, fallback: float) -> float: - if isinstance(ms, bool) or not isinstance(ms, int | float): - return fallback - try: - return float(ms) - except OverflowError: # an int too big for a float - return fallback - - -def _outcome(scratch: Path, box: BoxResult) -> tuple[object, str | None, float]: - if box.timed_out: - return None, "TimeLimitExceeded", box.elapsed_ms - payload = _read_result(scratch) - if payload is None: - return None, _crash(box), box.elapsed_ms - took = _took(payload.get("ms"), box.elapsed_ms) - if payload["ok"] is True: - return payload.get("value"), None, took - return None, str(payload.get("error", "Error"))[:_MAX_ERROR], took - - -@dataclass(frozen=True) -class _Job: - language: Language - binary: Path - code: str - entry: Entry - limits: Limits - box: Box - - -def _case(job: _Job, example: Example) -> CaseResult: - try: - args = _positional(example, job.entry) - except ValueError as exc: - return CaseResult( - example.n, example.args, example.expected, None, ok=False, error=str(exc) - ) - language = job.language - with tempfile.TemporaryDirectory(prefix="code-runner-") as tmp: - scratch = Path(tmp) - language.prepare(scratch, job.code, job.entry, args) - done = job.box( - language.command(job.binary), - scratch, - job.limits, - language.memory_bytes, - language.ro_binds(job.binary), - ) - actual, error, ms = _outcome(scratch, done) - ok = False - if error is None: - try: - ok = matches(example.expected, actual, example.order) - except (TooDeepError, RecursionError, MemoryError) as exc: - actual, error = None, f"InvalidResult: {type(exc).__name__}" - return CaseResult( - example.n, example.args, example.expected, actual, ok, done.stdout, error, ms - ) - - -def run( - lang: str, - code: str, - starter: str, - examples: Sequence[Example], - *, - limits: Limits | None = None, - box: Box = execute, -) -> list[CaseResult]: - language = get_language(lang) - entry = parse_starter(language.name, starter) - job = _Job(language, language.resolve(), code, entry, limits or Limits(), box) - return [_case(job, example) for example in examples] diff --git a/code_runner/code_runner/_sandbox.py b/code_runner/code_runner/_sandbox.py deleted file mode 100644 index 60c51b5..0000000 --- a/code_runner/code_runner/_sandbox.py +++ /dev/null @@ -1,147 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import functools -import math -import os -from pathlib import Path -import signal -import subprocess -import tempfile -import time -from typing import IO - -from code_runner._model import MAX_CAPTURE, Limits, RunnerError - -WORK = "/work" -PRLIMIT = "/usr/bin/prlimit" -_ROOT_LINKS = ("bin", "sbin", "lib", "lib64") - - -@dataclass(frozen=True) -class BoxResult: - returncode: int - timed_out: bool - stdout: str - stderr: str - elapsed_ms: float - - -def _root_links() -> list[str]: - args: list[str] = [] - for name in _ROOT_LINKS: - path = Path("/") / name - if path.is_symlink(): - args += ["--symlink", str(path.readlink()), str(path)] - elif path.is_dir(): - args += ["--ro-bind", str(path), str(path)] - return args - - -def _base_argv() -> list[str]: - return [ - "bwrap", - "--unshare-all", # no --share-net: no network at all, loopback included - "--die-with-parent", - "--new-session", - "--clearenv", - "--ro-bind", - "/usr", - "/usr", - *_root_links(), - "--proc", - "/proc", - "--dev", - "/dev", - "--tmpfs", - "/tmp", - ] - - -def bwrap_argv( - command: list[str], - scratch: Path, - limits: Limits, - memory_bytes: int, - ro_binds: tuple[str, ...] = (), -) -> list[str]: - argv = _base_argv() - for bind in ro_binds: - argv += ["--ro-bind", bind, bind] - argv += ["--bind", str(scratch), WORK, "--remount-ro", "/", "--chdir", WORK] - for name, value in ( - ("HOME", WORK), - ("PATH", "/usr/bin"), - ("LANG", "C.UTF-8"), - ("PYTHONIOENCODING", "utf-8"), - ): - argv += ["--setenv", name, value] - cpu = math.ceil(limits.timeout_s) + 1 - return [ - *argv, - "--", - PRLIMIT, - f"--as={memory_bytes}", - f"--nproc={limits.nproc}", - f"--fsize={limits.fsize_bytes}", - f"--nofile={limits.nofile}", - f"--cpu={cpu}", - "--core=0", - "--", - *command, - ] - - -@functools.cache -def check_sandbox() -> None: - if not Path(PRLIMIT).is_file(): - msg = f"{PRLIMIT} is missing (util-linux)" - raise RunnerError(msg) - try: - probe = subprocess.run( - [*_base_argv(), "/usr/bin/true"], - capture_output=True, - check=False, - timeout=20, - ) - except OSError as exc: - msg = f"bubblewrap (bwrap) cannot start: {exc}" - raise RunnerError(msg) from exc - if probe.returncode != 0: - detail = probe.stderr.decode(errors="replace").strip() - msg = f"bubblewrap cannot create the sandbox: {detail}" - raise RunnerError(msg) - - -def _read(handle: IO[bytes]) -> str: - handle.seek(0) - return handle.read(MAX_CAPTURE).decode("utf-8", errors="replace") - - -def execute( - command: list[str], - scratch: Path, - limits: Limits, - memory_bytes: int, - ro_binds: tuple[str, ...] = (), -) -> BoxResult: - check_sandbox() - argv = bwrap_argv(command, scratch, limits, memory_bytes, ro_binds) - with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: - started = time.monotonic() - with subprocess.Popen( - argv, - stdin=subprocess.DEVNULL, - stdout=out, - stderr=err, - env={"PATH": "/usr/bin:/bin"}, - start_new_session=True, - ) as proc: - timed_out = False - try: - proc.wait(timeout=limits.timeout_s) - except subprocess.TimeoutExpired: - timed_out = True - os.killpg(proc.pid, signal.SIGKILL) - elapsed = (time.monotonic() - started) * 1000 - return BoxResult(proc.returncode, timed_out, _read(out), _read(err), elapsed) diff --git a/code_runner/code_runner/_split.py b/code_runner/code_runner/_split.py deleted file mode 100644 index cacd2ee..0000000 --- a/code_runner/code_runner/_split.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -_OPEN = "([{<" -_CLOSE = ")]}>" - - -def split_top_level(text: str) -> list[str]: - parts: list[str] = [] - depth = 0 - quote = "" - escaped = False - start = 0 - for i, ch in enumerate(text): - if escaped: - escaped = False - elif quote: - escaped = ch == "\\" - quote = "" if ch == quote else quote - elif ch in "\"'": - quote = ch - elif ch in _OPEN: - depth += 1 - elif ch in _CLOSE and not (ch == ">" and text[i - 1] in "=-"): - depth = max(depth - 1, 0) - elif ch == "," and depth == 0: - parts.append(text[start:i]) - start = i + 1 - parts.append(text[start:]) - return [part.strip() for part in parts] diff --git a/code_runner/code_runner/_starter.py b/code_runner/code_runner/_starter.py deleted file mode 100644 index fa74ff0..0000000 --- a/code_runner/code_runner/_starter.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import re - -from code_runner._model import RunnerError -from code_runner._split import split_top_level - -_IDENT = r"[A-Za-z_$][\w$]*" -_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) -_LINE_COMMENT = re.compile(r"^\s*(?://|#).*$", re.MULTILINE) -_PY_CLASS = re.compile(r"^class\s+(\w+)", re.MULTILINE) -_PY_DEF = re.compile(r"\bdef\s+(?!__)(\w+)\s*\(") -_PY_FUNC = re.compile(r"^def\s+(\w+)\s*\(", re.MULTILINE) -_JS_CLASS = re.compile(rf"^\s*class\s+({_IDENT})", re.MULTILINE) -_JS_METHOD = re.compile( - rf"^\s+(?:async\s+)?(?!constructor\b)({_IDENT})\s*\(", re.MULTILINE -) -_JS_ASSIGNED = re.compile( - rf"\b(?:var|let|const)\s+({_IDENT})\s*(?::[^=]+)?=\s*(?:async\s+)?(?:function\b\s*\w*\s*)?\(" -) -_JS_DECLARED = re.compile(rf"\bfunction\s+({_IDENT})\s*\(") -_NOT_METHODS = {"if", "for", "while", "switch", "catch", "return", "function"} - - -@dataclass(frozen=True) -class Entry: - cls: str | None - name: str - params: tuple[str, ...] - - -def _no_entry(lang: str) -> RunnerError: - return RunnerError(f"no entry point found in the {lang} starter code") - - -def _balanced(text: str, open_at: int) -> str: - depth = 0 - for i in range(open_at, len(text)): - ch = text[i] - if ch in "([{" or (ch == "<" and depth > 0): - depth += 1 - elif ch in ")]}" or (ch == ">" and text[i - 1] != "=" and depth > 1): - depth -= 1 - if depth == 0: - return text[open_at + 1 : i] - return text[open_at + 1 :] - - -def _params(code: str, open_at: int) -> tuple[str, ...]: - names: list[str] = [] - for part in split_top_level(_balanced(code, open_at)): - found = re.match(_IDENT, part.strip()) - if found: - names.append(found.group()) - return tuple(names) - - -def _python(text: str) -> Entry | None: - code = _LINE_COMMENT.sub("", text) - owner = _PY_CLASS.search(code) - if owner is None: - found = _PY_FUNC.search(code) - return found and Entry(None, found.group(1), _params(code, found.end() - 1)) - found = _PY_DEF.search(code, owner.end()) - if found is None: - return None - params = _params(code, found.end() - 1) - return Entry( - owner.group(1), found.group(1), tuple(p for p in params if p != "self") - ) - - -def _javascript(text: str) -> Entry | None: - code = _LINE_COMMENT.sub("", _BLOCK_COMMENT.sub("", text)) - owner = _JS_CLASS.search(code) - if owner is not None: - for method in _JS_METHOD.finditer(code, owner.end()): - if method.group(1) not in _NOT_METHODS: - return Entry( - owner.group(1), method.group(1), _params(code, method.end() - 1) - ) - return None - found = _JS_ASSIGNED.search(code) or _JS_DECLARED.search(code) - return found and Entry(None, found.group(1), _params(code, found.end() - 1)) - - -def parse_starter(lang: str, text: str) -> Entry: - entry = _python(text) if lang == "python3" else _javascript(text) - if entry is None: - raise _no_entry(lang) - return entry - - -def extract_starter(statement: str) -> str: - lines = statement.splitlines(keepends=True) - heads = [i for i, line in enumerate(lines) if line.startswith("## Starter code")] - if not heads: - return "" - body = lines[heads[0] + 1 :] - if body and body[0].startswith("```"): - body = body[1:] - end = next( - (i for i, line in enumerate(body) if line.startswith("```")), len(body) - ) - return "".join(body[:end]) - end = next((i for i, line in enumerate(body) if line.startswith("## ")), len(body)) - return "".join(body[:end]) diff --git a/code_runner/code_runner/harness/harness.js b/code_runner/code_runner/harness/harness.js deleted file mode 100644 index ab975af..0000000 --- a/code_runner/code_runner/harness/harness.js +++ /dev/null @@ -1,45 +0,0 @@ -"use strict"; -// Runs inside the sandbox: call the solution once, write result.json. -const fs = require("node:fs"); -const path = require("node:path"); - -const root = process.argv[2]; -const spec = JSON.parse(fs.readFileSync(path.join(root, "input.json"), "utf8")); -const IDENT = /^[A-Za-z_$][\w$]*$/; - -function describe(e) { - const name = e && typeof e.name === "string" ? e.name : "Error"; - const message = e && e.message !== undefined ? e.message : String(e); - return `${name}: ${message}`.slice(0, 300); -} - -function writeResult(payload) { - // Unique name + rename: a symlink planted at result.json is replaced, never followed. - const unique = path.join(root, `.result-${process.pid}-${Math.random().toString(36).slice(2)}`); - fs.writeFileSync(unique, JSON.stringify(payload), { flag: "wx" }); - fs.renameSync(unique, path.join(root, "result.json")); -} - -function load(target) { - if (!IDENT.test(target)) throw new TypeError("bad entry name"); - if (spec.typescript) return require(path.join(root, "solution.cts")).__entry; - const source = fs.readFileSync(path.join(root, "solution.js"), "utf8"); - return new Function(`${source}\n;return ${target};`)(); -} - -function call() { - const entry = load(spec.cls || spec.fn); - if (spec.cls) return new entry()[spec.fn](...spec.args); - return entry(...spec.args); -} - -const started = process.hrtime.bigint(); -let payload; -try { - const value = call(); - payload = { ok: true, value: value === undefined ? null : JSON.parse(JSON.stringify(value)) }; -} catch (e) { - payload = { ok: false, error: describe(e) }; -} -payload.ms = Number(process.hrtime.bigint() - started) / 1e6; -writeResult(payload); diff --git a/code_runner/code_runner/harness/harness.py b/code_runner/code_runner/harness/harness.py deleted file mode 100644 index f04c8a8..0000000 --- a/code_runner/code_runner/harness/harness.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Runs inside the sandbox: call the solution once, write result.json.""" - -from __future__ import annotations - -import json -import os -from pathlib import Path -import sys -import tempfile -import time -from typing import Any - -PRELUDE = ( - "from typing import *\n" - "import bisect, collections, functools, heapq, itertools, math\n" - "import operator, re, string\n" -) -MAX_MESSAGE = 300 - - -def describe(exc: BaseException) -> str: - return f"{type(exc).__name__}: {exc}"[:MAX_MESSAGE] - - -def write_result(root: Path, payload: dict[str, object]) -> None: - # Unique name + rename: a symlink the solution planted at result.json is - # replaced by this file, never followed. - handle, name = tempfile.mkstemp(dir=root, prefix=".result-") - with os.fdopen(handle, "w", encoding="utf-8") as out: - json.dump(payload, out) - Path(name).replace(root / "result.json") - - -def call(root: Path, spec: dict[str, Any]) -> object: - namespace: dict[str, Any] = {"__name__": "solution"} - exec(PRELUDE, namespace) - source = (root / "solution.py").read_text(encoding="utf-8") - exec(compile(source, "solution.py", "exec"), namespace) - owner, name = spec["cls"], str(spec["fn"]) - if owner is None: - target = namespace.get(name) - else: - cls = namespace.get(str(owner)) - target = getattr(cls() if cls else None, name, None) - if target is None: - message = f"name {owner or name!r} is not defined" - raise NameError(message) - return target(*spec["args"]) - - -def main(directory: str) -> None: - root = Path(directory) - spec = json.loads((root / "input.json").read_text(encoding="utf-8")) - started = time.perf_counter() - try: - value = call(root, spec) - payload: dict[str, object] = { - "ok": True, - "value": json.loads(json.dumps(value)), - } - except BaseException as exc: - payload = {"ok": False, "error": describe(exc)} - payload["ms"] = (time.perf_counter() - started) * 1000 - sys.stdout.flush() - write_result(root, payload) - - -if __name__ == "__main__": - main(sys.argv[1]) diff --git a/code_runner/code_runner/py.typed b/code_runner/code_runner/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/code_runner/code_runner/tests/__init__.py b/code_runner/code_runner/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/code_runner/code_runner/tests/conftest.py b/code_runner/code_runner/tests/conftest.py deleted file mode 100644 index d55c3eb..0000000 --- a/code_runner/code_runner/tests/conftest.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Shared fixtures: real-sandbox tests are skipped where bubblewrap cannot run.""" - -from __future__ import annotations - -import shutil - -import pytest - -from code_runner._model import Example, RunnerError -from code_runner._sandbox import check_sandbox - - -def _bwrap_works() -> bool: - try: - check_sandbox() - except RunnerError: - return False - return True - - -needs_sandbox = pytest.mark.skipif(not _bwrap_works(), reason="bwrap/userns unusable") -needs_node = pytest.mark.skipif( - not _bwrap_works() or not shutil.which("node"), reason="node or bwrap missing" -) - -TWO_SUM_STARTER = { - "python3": "class Solution:\n def twoSum(self, nums: list[int], target: int) -> list[int]:\n", - "javascript": "var twoSum = function(nums, target) {\n\n};", - "typescript": "function twoSum(nums: number[], target: number): number[] {\n};", -} -TWO_SUM_CODE = { - "python3": ( - "class Solution:\n" - " def twoSum(self, nums, target):\n" - " seen = {}\n" - " for i, n in enumerate(nums):\n" - " if target - n in seen:\n" - " return [i, seen[target - n]]\n" - " seen[n] = i\n" - ), - "javascript": ( - "var twoSum = function(nums, target) {\n" - " const seen = new Map();\n" - " for (let i = 0; i < nums.length; i++) {\n" - " if (seen.has(target - nums[i])) return [i, seen.get(target - nums[i])];\n" - " seen.set(nums[i], i);\n" - " }\n" - "};" - ), - "typescript": ( - "function twoSum(nums: number[], target: number): number[] {\n" - " const seen = new Map();\n" - " for (let i = 0; i < nums.length; i++) {\n" - " const j = seen.get(target - nums[i]);\n" - " if (j !== undefined) return [i, j];\n" - " seen.set(nums[i], i);\n" - " }\n" - " return [];\n" - "}" - ), -} - - -@pytest.fixture -def two_sum_examples() -> list[Example]: - return [ - Example(1, {"nums": [2, 7, 11, 15], "target": 9}, [0, 1], "any"), - Example(2, {"nums": [3, 2, 4], "target": 6}, [1, 2], "any"), - Example(3, {"nums": [3, 3], "target": 6}, [0, 1], "any"), - ] diff --git a/code_runner/code_runner/tests/test_compare.py b/code_runner/code_runner/tests/test_compare.py deleted file mode 100644 index 64e5efd..0000000 --- a/code_runner/code_runner/tests/test_compare.py +++ /dev/null @@ -1,65 +0,0 @@ -import pytest - -from code_runner._compare import MAX_DEPTH, TooDeepError, matches, too_deep - - -@pytest.mark.parametrize( - ("expected", "actual", "order", "want"), - [ - ([0, 1], [0, 1], "exact", True), - ([0, 1], [1, 0], "exact", False), - ([0, 1], [1, 0], "any", True), - ([0, 1], [1, 0, 2], "any", False), - ([[1, 2], [3]], [[3], [1, 2]], "any", True), - ([[1, 2], [3]], [[3], [2, 1]], "any", False), - ([[1, 2], [3]], [[3], [2, 1]], "deep", True), - ([[1, 2], [3]], [[3], [2, 9]], "deep", False), - (3, 3.0, "exact", True), - (0.1 + 0.2, 0.3, "exact", True), - (2.0, 2.001, "exact", False), - (True, 1, "exact", False), - (1, True, "exact", False), - (True, True, "exact", True), - (None, None, "exact", True), - (None, 0, "exact", False), - ("ab", "ab", "exact", True), - ("ab", "ba", "exact", False), - ("1", 1, "exact", False), - ({"a": [1]}, {"a": [1]}, "exact", True), - ({"a": 1}, {"b": 1}, "exact", False), - ({"a": 1}, {"a": 2}, "exact", False), - (5, 5, "any", True), - ([1], 1, "any", False), - ([1.0, 2.0], [2.0, 1.0000001], "any", True), - ], -) -def test_matches(expected: object, actual: object, order: str, *, want: bool) -> None: - assert matches(expected, actual, order) is want - - -def test_unknown_order_is_refused() -> None: - with pytest.raises(ValueError, match="order"): - matches(1, 1, "sideways") - - -def _nest(depth: int) -> object: - value: object = 1 - for _ in range(depth): - value = [value] - return value - - -@pytest.mark.parametrize("order", ["exact", "any", "deep"]) -def test_deep_nesting_is_refused_not_recursed(order: str) -> None: - deep = _nest(100_000) # would raise RecursionError in the recursive walks - with pytest.raises(TooDeepError): - matches(deep, deep, order) - with pytest.raises(TooDeepError): - matches(1, deep, order) - - -def test_nesting_up_to_the_limit_still_compares() -> None: - assert matches(_nest(MAX_DEPTH), _nest(MAX_DEPTH), "deep") is True - assert too_deep(_nest(MAX_DEPTH + 1)) is True - assert too_deep({"a": [{"b": _nest(MAX_DEPTH)}]}) is True - assert too_deep({"a": [{"b": 1}]}) is False diff --git a/code_runner/code_runner/tests/test_examples.py b/code_runner/code_runner/tests/test_examples.py deleted file mode 100644 index f7622e5..0000000 --- a/code_runner/code_runner/tests/test_examples.py +++ /dev/null @@ -1,106 +0,0 @@ -from code_runner._examples import parse_examples -from code_runner._model import Example - -TWO_SUM = """# 1. Two Sum (Easy) -Tags: Array, Hash Table - -Return indices. - -You can return the answer in any order. - -**Example 1:** - -**Input:** `nums = [2,7,11,15], target = 9` - -**Output:** `[0,1]` - -**Explanation:** 2 + 7. - -**Example 2:** - -**Input:** `nums = [3,2,4], target = 6` - -**Output:** `[1,2]` - -## Starter code (python3) -``` -class Solution: - def twoSum(self, nums, target): -``` -""" - -FENCED = """# 5. Pal - -Output order does not matter. - -**Example 1:** -``` -Input: s = "a,b", flag = true -Output: "b" -Explanation: Output: not an example -``` -**Example 2:** -``` -Input: s = 'x', flag = false -Output: None -``` -""" - -THREE_SUM = """# 15. 3Sum -Notice that the order of the output and the order of the triplets does not matter. -**Example 1:** -**Input:** nums = [-1,0,1,2,-1,-4] -**Output:** [[-1,-1,2],[-1,0,1]] -""" - -SKIPPED = """# 2. Add Two Numbers -**Example 1:** -**Input:** l1 = 1->2 -**Output:** [1,2] -**Example 2:** -**Input:** a = 1, b = 2 -**Output:** 3.00000 -**Example 3:** -**Input:** a = 1 -**Output:** 1->2 -**Example 4:** -**Input:** a -**Output:** 1 -""" - - -def test_two_sum_examples_are_typed_and_ordered_any() -> None: - found = parse_examples(TWO_SUM) - assert found == [ - Example(1, {"nums": [2, 7, 11, 15], "target": 9}, [0, 1], "any"), - Example(2, {"nums": [3, 2, 4], "target": 6}, [1, 2], "any"), - ] - - -def test_fenced_examples_and_python_literals() -> None: - found = parse_examples(FENCED) - assert [e.args for e in found] == [ - {"s": "a,b", "flag": True}, - {"s": "x", "flag": False}, - ] - assert [e.expected for e in found] == ["b", None] - assert {e.order for e in found} == {"exact"} - - -def test_triplets_sentence_means_deep_order() -> None: - assert parse_examples(THREE_SUM)[0].order == "deep" - - -def test_unparseable_examples_are_skipped_but_numbering_is_kept() -> None: - found = parse_examples(SKIPPED) - assert found == [Example(2, {"a": 1, "b": 2}, 3.0, "exact")] - - -def test_output_without_input_and_empty_statement() -> None: - assert parse_examples("**Output:** 1\n") == [] - assert parse_examples("") == [] - - -def test_starter_section_is_never_scanned() -> None: - text = "# t\n## Starter code (python3)\n```\nInput: a = 1\nOutput: 2\n```\n" - assert parse_examples(text) == [] diff --git a/code_runner/code_runner/tests/test_internals.py b/code_runner/code_runner/tests/test_internals.py deleted file mode 100644 index 7172f86..0000000 --- a/code_runner/code_runner/tests/test_internals.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Branches that a healthy sandbox never reaches: crashes, bad results, no bwrap.""" - -from __future__ import annotations - -import dataclasses -import os -from pathlib import Path -import subprocess -from typing import TYPE_CHECKING - -import pytest - -from code_runner import Example, Limits, RunnerError, run -from code_runner._examples import parse_examples -from code_runner._languages import JAVASCRIPT, PYTHON, get_language -from code_runner._run import Box, _read_result -from code_runner._sandbox import BoxResult, bwrap_argv, check_sandbox -from code_runner._starter import parse_starter - -if TYPE_CHECKING: - from collections.abc import Iterator - -STARTER = "def f(a):\n" -EXAMPLE = Example(1, {"a": 1}, 1) - - -@pytest.fixture(autouse=True) -def _fresh_sandbox_check() -> Iterator[None]: - check_sandbox.cache_clear() - yield - check_sandbox.cache_clear() - - -def _fake(result: BoxResult, files: dict[str, str] | None = None) -> Box: - def box( - command: list[str], - scratch: Path, - limits: Limits, - memory: int, - binds: tuple[str, ...], - ) -> BoxResult: - for name, text in (files or {}).items(): - (scratch / name).write_text(text) - return result - - return box - - -def _box(**kw: object) -> BoxResult: - fields = { - "returncode": 0, - "timed_out": False, - "stdout": "", - "stderr": "", - "elapsed_ms": 7.0, - } - return BoxResult(**(fields | kw)) - - -def _run(box: Box) -> object: - return run("python3", "", STARTER, [EXAMPLE], box=box)[0] - - -def test_crash_without_result_names_the_exit_code() -> None: - result = _run(_fake(_box(returncode=139))) - assert (result.ok, result.actual, result.error, result.ms) == ( - False, - None, - "Crashed (exit 139)", - 7.0, - ) - - -def test_out_of_memory_crash_is_named() -> None: - stderr = "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory" - assert ( - _run(_fake(_box(returncode=134, stderr=stderr))).error == "MemoryLimitExceeded" - ) - - -def test_timeout_wins_over_a_result_file() -> None: - files = {"result.json": '{"ok": true, "value": 1}'} - result = _run(_fake(_box(timed_out=True, elapsed_ms=1000.0), files)) - assert (result.error, result.ms) == ("TimeLimitExceeded", 1000.0) - - -def test_result_file_variants() -> None: - good = _run( - _fake(_box(stdout="hi"), {"result.json": '{"ok": true, "value": 1, "ms": 2.5}'}) - ) - assert (good.ok, good.ms, good.stdout) == (True, 2.5, "hi") - no_ms = _run(_fake(_box(), {"result.json": '{"ok": true, "value": 1}'})) - assert no_ms.ms == 7.0 - failed = _run(_fake(_box(), {"result.json": '{"ok": false}'})) - assert failed.error == "Error" - named = _run( - _fake(_box(), {"result.json": '{"ok": false, "error": "KeyError: x"}'}) - ) - assert named.error == "KeyError: x" - - -def test_hostile_result_files_cost_one_example_not_the_run() -> None: - hostile = { - "bomb": "[" * 200_000, - "deep": '{"ok": true, "value": ' + "[" * 5000 + "]" * 5000 + "}", - "huge_ms": '{"ok": true, "value": 1, "ms": 1' + "0" * 400 + "}", - "bool_ms": '{"ok": true, "value": 1, "ms": true}', - "long_error": '{"ok": false, "error": "' + "x" * 5000 + '"}', - } - results = { - name: _run(_fake(_box(), {"result.json": text})) - for name, text in hostile.items() - } - assert results["bomb"].error == "Crashed (exit 0)" - assert results["deep"].error == "Crashed (exit 0)" - assert (results["huge_ms"].ok, results["huge_ms"].ms) == (True, 7.0) - assert results["bool_ms"].ms == 7.0 - assert len(results["long_error"].error) == 200 - - -def test_too_deep_expected_or_actual_is_that_examples_error() -> None: - deep: object = 1 - for _ in range(100): - deep = [deep] - files = {"result.json": '{"ok": true, "value": 1}'} - cases = [Example(1, {"a": 1}, deep), EXAMPLE] - results = run("python3", "", STARTER, cases, box=_fake(_box(), files)) - assert (results[0].ok, results[0].actual) == (False, None) - assert results[0].error == "InvalidResult: TooDeepError" - assert results[1].ok is True # the run went on - - -def test_recursion_in_compare_is_that_examples_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def boom(*args: object) -> bool: - raise RecursionError - - monkeypatch.setattr("code_runner._run.matches", boom) - files = {"result.json": '{"ok": true, "value": 1}'} - result = _run(_fake(_box(), files)) - assert result.error == "InvalidResult: RecursionError" - - -def test_unreadable_result_files_are_a_crash(tmp_path: Path) -> None: - secret = tmp_path / "secret" - secret.write_text('{"ok": true, "value": "stolen"}') - (tmp_path / "result.json").symlink_to(secret) - assert _read_result(tmp_path) is None - (tmp_path / "result.json").unlink() - os.mkfifo(tmp_path / "result.json") - assert _read_result(tmp_path) is None - (tmp_path / "result.json").unlink() - for text in ("not json", "[1]", '{"no": "ok"}'): - (tmp_path / "result.json").write_text(text) - assert _read_result(tmp_path) is None - assert _read_result(tmp_path / "missing") is None - - -def test_binary_resolution(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - ghost = dataclasses.replace( - PYTHON, binary_default="/nope", program="no-such-program-xyz" - ) - monkeypatch.delenv(ghost.binary_env, raising=False) - with pytest.raises(RunnerError, match="no-such-program-xyz not found"): - ghost.resolve() - tool = tmp_path / "bin" / "python3" - tool.parent.mkdir() - tool.write_text("#!/bin/sh\n") - monkeypatch.setenv(ghost.binary_env, str(tool)) - assert ghost.resolve() == tool - assert ghost.ro_binds(tool) == (str(tool.parent.parent),) - assert PYTHON.ro_binds(Path("/usr/bin/python3")) == () - assert get_language("TS").name == "typescript" - - -def _has(argv: list[str], seq: list[str]) -> bool: - return any(argv[i : i + len(seq)] == seq for i in range(len(argv))) - - -def test_argv_binds_links_and_limits( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr("code_runner._sandbox._ROOT_LINKS", ("usr", "no-such-root-dir")) - argv = bwrap_argv(["prog"], tmp_path, Limits(timeout_s=1.2), 123, ("/opt/x",)) - assert _has(argv, ["--ro-bind", "/usr", "/usr"]) - assert _has(argv, ["--ro-bind", "/opt/x", "/opt/x"]) - assert _has(argv, ["--bind", str(tmp_path), "/work"]) - assert "--share-net" not in argv - assert {"--unshare-all", "--die-with-parent", "--clearenv", "--remount-ro"} <= set( - argv - ) - assert "--as=123" in argv - assert "--cpu=3" in argv - assert argv[-1] == "prog" - - -def test_sandbox_preflight_failures(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("code_runner._sandbox.PRLIMIT", "/no/such/prlimit") - with pytest.raises(RunnerError, match="prlimit"): - check_sandbox() - monkeypatch.undo() - - def missing(*args: object, **kwargs: object) -> None: - message = "bwrap" - raise FileNotFoundError(message) - - monkeypatch.setattr(subprocess, "run", missing) - with pytest.raises(RunnerError, match="cannot start"): - check_sandbox() - - def refused(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: - return subprocess.CompletedProcess(args, 1, b"", b"bwrap: No permissions\n") - - monkeypatch.setattr(subprocess, "run", refused) - with pytest.raises(RunnerError, match="No permissions"): - check_sandbox() - - -def test_statement_and_starter_edge_cases() -> None: - escaped = parse_examples('**Input:** s = "a\\"b,c", t = 1\n**Output:** "x"\n') - assert escaped[0].args == {"s": 'a"b,c', "t": 1} - assert parse_starter("python3", "def f(a, b").params == ("a", "b") - skipped = "class S {\n if (x) {\n solve(a) {\n }\n}\n" - assert parse_starter("javascript", skipped).name == "solve" - - -def test_typescript_class_entry_is_exported(tmp_path: Path) -> None: - entry = parse_starter( - "typescript", "class Solution {\n solve(a: number): number {\n }\n}\n" - ) - get_language("typescript").prepare(tmp_path, "class Solution {}", entry, [1]) - assert "__entry: Solution" in (tmp_path / "solution.cts").read_text() - JAVASCRIPT.prepare(tmp_path, "x", entry, [1]) - assert (tmp_path / "solution.js").read_text() == "x" diff --git a/code_runner/code_runner/tests/test_isolation.py b/code_runner/code_runner/tests/test_isolation.py deleted file mode 100644 index bb877e5..0000000 --- a/code_runner/code_runner/tests/test_isolation.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Escape attempts: each one runs hostile code in the real sandbox and must fail.""" - -from __future__ import annotations - -from pathlib import Path -import socket -import subprocess -import time -from typing import Any, cast - -import pytest - -from code_runner import CaseResult, Example, Limits, run -from code_runner.tests.conftest import needs_node, needs_sandbox - -pytestmark = needs_sandbox -STARTER = "def f(a):\n" -FAST = Limits(timeout_s=1.0) - - -def _one(code: str, lang: str = "python3", limits: Limits = FAST) -> CaseResult: - starter = {"python3": STARTER, "javascript": "var f = function(a) {};"}[lang] - return run(lang, code, starter, [Example(1, {"a": 0}, 1)], limits=limits)[0] - - -def _pgrep(pattern: str) -> int: - done = subprocess.run( - ["/usr/bin/pgrep", "-fc", pattern], capture_output=True, text=True, check=False - ) - return int(done.stdout.strip() or 0) - - -def test_no_network_not_even_host_loopback() -> None: - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - listener.listen() - listener.settimeout(0) - port = listener.getsockname()[1] - code = ( - "import socket\n" - "def f(a):\n" - " out = []\n" - f" for host, port in [('127.0.0.1', {port}), ('1.1.1.1', 53), ('::1', {port})]:\n" - " try:\n" - " socket.create_connection((host, port), timeout=1).close()\n" - " out.append('connected')\n" - " except OSError:\n" - " out.append('blocked')\n" - " return out\n" - ) - result = _one(code) - with pytest.raises(BlockingIOError): - listener.accept() - assert result.actual == ["blocked", "blocked", "blocked"] - - -@needs_node -def test_no_network_from_node() -> None: - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - listener.listen() - port = listener.getsockname()[1] - code = ( - "var f = function(a) {\n" - " const net = process.getBuiltinModule('node:net');\n" - " let seen = 'pending';\n" - f" const s = net.connect({port}, '127.0.0.1');\n" - " s.on('connect', () => { seen = 'connected'; });\n" - " s.on('error', () => {});\n" - " return seen;\n" - "};" - ) - result = _one(code, "javascript") - listener.settimeout(0.2) - with pytest.raises(TimeoutError): - listener.accept() - assert result.actual == "pending" - - -def test_writes_outside_scratch_never_reach_the_host(tmp_path: Path) -> None: - marker = tmp_path / "marker" - code = ( - "def f(a):\n" - " out = []\n" - f" for path in ['/usr/escape', '/etc/escape', '/escape', {str(marker)!r}]:\n" - " try:\n" - " open(path, 'w').write('x')\n" - " out.append('wrote')\n" - " except OSError:\n" - " out.append('denied')\n" - " return out\n" - ) - result = _one(code) - assert result.actual == ["denied", "denied", "denied", "denied"] - assert not marker.exists() - assert not Path("/usr/escape").exists() - - -def test_host_files_and_environment_are_invisible( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - secret = tmp_path / "secret.txt" - secret.write_text("hunter2") - monkeypatch.setenv("CODE_RUNNER_SECRET", "hunter2") - code = ( - "import os\n" - "def f(a):\n" - f" return [os.path.exists({str(secret)!r}), os.path.exists({str(Path.home())!r}),\n" - " 'CODE_RUNNER_SECRET' in os.environ, sorted(os.environ), os.getpid() < 20,\n" - " os.path.exists('/proc/1/environ') and 'hunter2' in open('/proc/1/environ').read()]\n" - ) - values = cast("list[Any]", _one(code).actual) - exists, home, env_leak, names, own_pidns, proc_leak = values - assert (exists, home, env_leak, own_pidns, proc_leak) == ( - False, - False, - False, - True, - False, - ) - assert set(names) <= {"HOME", "LANG", "PATH", "PWD", "PYTHONIOENCODING"} - - -def test_symlink_in_scratch_cannot_exfiltrate_a_host_file(tmp_path: Path) -> None: - secret = tmp_path / "secret.txt" - secret.write_text("hunter2-secret-contents") - code = ( - "import os\n" - "def f(a):\n" - f" os.symlink({str(secret)!r}, '/work/result.json')\n" - " return 7\n" - ) - result = _one(code) - assert "hunter2" not in repr(result) - assert result.actual == 7 # the harness replaced the link, never followed it - - -def test_fork_bomb_is_contained_and_leaves_nothing_behind() -> None: - before = _pgrep("python3 -I /work/harness.py") - code = ( - "import os, subprocess\n" - "def f(a):\n" - " subprocess.Popen(['sleep', '97'])\n" - " while True:\n" - " os.fork()\n" - ) - started = time.monotonic() - result = _one(code) - assert time.monotonic() - started < 6 - assert result.ok is False - assert result.error is not None - assert result.error.startswith(("TimeLimitExceeded", "BlockingIOError")) - time.sleep(0.3) - assert _pgrep("python3 -I /work/harness.py") == before - assert _pgrep("^sleep 97$") == 0 - - -def test_memory_bomb_is_a_visible_error() -> None: - result = _one("def f(a):\n return len(bytearray(8 * 1024**3))\n") - assert result.ok is False - assert result.error is not None - assert result.error.startswith("MemoryError") - - -@needs_node -def test_node_heap_bomb_is_a_visible_error() -> None: - code = "var f = function(a) { const x = []; for (;;) x.push(new Array(1e6).fill(1)); };" - result = _one(code, "javascript", Limits(timeout_s=5.0)) - assert result.ok is False - assert result.error in {"MemoryLimitExceeded", "RangeError: Invalid array length"} - - -def test_output_flood_is_capped() -> None: - code = "def f(a):\n while True:\n print('x' * 1000)\n" - started = time.monotonic() - result = _one(code) - assert time.monotonic() - started < 6 - assert result.ok is False - assert len(result.stdout) <= 64 * 1024 - - -def test_disk_fill_is_capped_by_the_file_size_limit() -> None: - code = ( - "def f(a):\n" - " with open('/work/big', 'wb') as fh:\n" - " while True:\n" - " fh.write(b'x' * 65536)\n" - ) - result = _one(code) - assert result.ok is False - assert result.error is not None - assert "OSError" in result.error - - -def test_timeout_kills_every_process_the_solution_started() -> None: - code = ( - "import subprocess\n" - "def f(a):\n" - " subprocess.Popen(['sleep', '97'])\n" - " while True:\n" - " pass\n" - ) - result = _one(code) - assert result.error == "TimeLimitExceeded" - time.sleep(0.3) - assert _pgrep("^sleep 97$") == 0 diff --git a/code_runner/code_runner/tests/test_run.py b/code_runner/code_runner/tests/test_run.py deleted file mode 100644 index c6a6ef4..0000000 --- a/code_runner/code_runner/tests/test_run.py +++ /dev/null @@ -1,227 +0,0 @@ -"""End to end through the real sandbox: every language, every kind of outcome.""" - -from __future__ import annotations - -import pytest - -from code_runner import CaseResult, Example, Limits, RunnerError, run -from code_runner.tests.conftest import ( - TWO_SUM_CODE, - TWO_SUM_STARTER, - needs_node, - needs_sandbox, -) - -LANGS = [ - pytest.param("python3", marks=needs_sandbox), - pytest.param("javascript", marks=needs_node), - pytest.param("typescript", marks=needs_node), -] -FAST = Limits(timeout_s=1.0) - - -def _two_sum(lang: str, code: str | None, examples: list[Example]) -> list[CaseResult]: - return run(lang, code or TWO_SUM_CODE[lang], TWO_SUM_STARTER[lang], examples) - - -@pytest.mark.parametrize("lang", LANGS) -def test_correct_solution_passes_every_example( - lang: str, two_sum_examples: list[Example] -) -> None: - results = _two_sum(lang, None, two_sum_examples) - assert [r.ok for r in results] == [True, True, True] - assert [r.n for r in results] == [1, 2, 3] - first = results[0] - assert first.input == {"nums": [2, 7, 11, 15], "target": 9} - assert first.expected == [0, 1] - assert sorted(first.actual) == [0, 1] # any order: [1, 0] is fine - assert first.error is None - assert first.ms >= 0 - - -WRONG = { - "python3": "class Solution:\n def twoSum(self, nums, target):\n return [0, 0]\n", - "javascript": "var twoSum = function(nums, target) { return [0, 0]; };", - "typescript": "function twoSum(nums: number[], target: number): number[] { return [0, 0]; }", -} - - -@pytest.mark.parametrize("lang", LANGS) -def test_wrong_answer_is_reported_with_actual_value( - lang: str, two_sum_examples: list[Example] -) -> None: - results = _two_sum(lang, WRONG[lang], two_sum_examples) - assert [r.ok for r in results] == [False, False, False] - assert results[0].actual == [0, 0] - assert results[0].error is None - - -THROWS = { - "python3": "class Solution:\n def twoSum(self, nums, target):\n return nums[99]\n", - "javascript": "var twoSum = function(nums, target) { throw new RangeError('boom'); };", - "typescript": "function twoSum(nums: number[], target: number): number[] { throw new RangeError('boom'); }", -} - - -@pytest.mark.parametrize("lang", LANGS) -def test_exception_becomes_an_error_per_case( - lang: str, two_sum_examples: list[Example] -) -> None: - results = _two_sum(lang, THROWS[lang], two_sum_examples[:1]) - only = results[0] - assert only.ok is False - assert only.actual is None - assert only.error is not None - assert only.error.split(":")[0] in {"IndexError", "RangeError"} - - -BROKEN = { - "python3": "class Solution:\n def twoSum(self, nums, target) return 1\n", - "javascript": "var twoSum = function(nums, target) { return ; ;; }}", - "typescript": "function twoSum(nums: number[], target: number): number[] { return [ }", -} - - -@pytest.mark.parametrize("lang", LANGS) -def test_syntax_error_is_reported(lang: str, two_sum_examples: list[Example]) -> None: - results = _two_sum(lang, BROKEN[lang], two_sum_examples[:2]) - assert [r.ok for r in results] == [False, False] - assert all(r.error and r.error.startswith("SyntaxError") for r in results) - - -PRINTS = { - "python3": "class Solution:\n def twoSum(self, nums, target):\n print('hi', target)\n return [0, 1]\n", - "javascript": "var twoSum = function(nums, target) { console.log('hi', target); return [0, 1]; };", - "typescript": "function twoSum(nums: number[], target: number): number[] { console.log('hi', target); return [0, 1]; }", -} - - -@pytest.mark.parametrize("lang", LANGS) -def test_prints_land_in_stdout_per_case( - lang: str, two_sum_examples: list[Example] -) -> None: - results = _two_sum(lang, PRINTS[lang], two_sum_examples[:2]) - assert results[0].stdout.strip() == "hi 9" - assert results[1].stdout.strip() == "hi 6" - - -@pytest.mark.parametrize("lang", LANGS) -def test_missing_entry_point_in_code_is_a_case_error( - lang: str, two_sum_examples: list[Example] -) -> None: - code = { - "python3": "x = 1\n", - "javascript": "var other = 1;", - "typescript": "const other = 1;", - } - results = _two_sum(lang, code[lang], two_sum_examples[:1]) - assert results[0].ok is False - assert results[0].error is not None - assert "twoSum" in results[0].error or "Solution" in results[0].error - - -def test_class_style_javascript_and_function_style_python() -> None: - js = run( - "javascript", - "class Solution { solve(a, b) { return a + b; } }", - "class Solution {\n solve(a, b) {\n }\n}", - [Example(1, {"a": 1, "b": 2}, 3)], - ) - py = run( - "python3", - "def add(a, b):\n return a + b\n", - "def add(a: int, b: int) -> int:\n", - [Example(1, {"a": 1, "b": 2}, 3)], - ) - assert js[0].ok is True - assert py[0].ok is True - - -def test_python_prelude_and_aliases() -> None: - code = "class Solution:\n def f(self, xs: List[int]) -> int:\n return collections.Counter(xs)[1]\n" - starter = "class Solution:\n def f(self, xs: List[int]) -> int:\n" - out = run("python", code, starter, [Example(1, {"xs": [1, 1, 2]}, 2)]) - assert out[0].ok is True - - -def test_positional_fallback_when_statement_names_differ() -> None: - out = run( - "python3", - "def add(a, b):\n return a - b\n", - "def add(a, b):\n", - [Example(1, {"x": 5, "y": 2}, 3)], - ) - assert out[0].ok is True - - -def test_argument_count_mismatch_is_a_case_error_without_running() -> None: - out = run( - "python3", - "def add(a, b):\n return a\n", - "def add(a, b):\n", - [Example(1, {"only": 5}, 3)], - ) - assert out[0].ok is False - assert out[0].error is not None - assert out[0].error.startswith("ArgumentError") - - -def test_non_json_result_is_a_case_error() -> None: - out = run( - "python3", - "def f(a):\n return {1, 2}\n", - "def f(a):\n", - [Example(1, {"a": 1}, [1, 2])], - ) - assert out[0].error is not None - assert out[0].error.startswith("TypeError") - - -def test_deep_order_and_float_tolerance() -> None: - out = run( - "python3", - "def f(a):\n return [[2, 1], [0, -1]]\n", - "def f(a):\n", - [Example(1, {"a": 0}, [[-1, 0], [1, 2]], "deep")], - ) - assert out[0].ok is True - - -def test_timeout_is_per_case() -> None: - out = run( - "python3", - "def f(a):\n while a:\n pass\n return 1\n", - "def f(a):\n", - [Example(1, {"a": 1}, 1), Example(2, {"a": 0}, 1)], - limits=FAST, - ) - assert out[0].error == "TimeLimitExceeded" - assert out[0].ok is False - assert out[0].ms >= 900 - assert out[1].ok is True - - -def test_unsupported_language_and_empty_examples() -> None: - with pytest.raises(RunnerError, match="language"): - run("cobol", "", "", []) - assert run("python3", "def f(a): pass\n", "def f(a):\n", []) == [] - - -@needs_sandbox -def test_json_bomb_result_costs_that_example_only() -> None: - code = ( - "import os\n" - "def f(a):\n" - " if a:\n" - " open('/work/result.json', 'w').write('[' * 200000)\n" - " os._exit(0)\n" - " return 1\n" - ) - out = run( - "python3", - code, - "def f(a):\n", - [Example(1, {"a": 1}, 1), Example(2, {"a": 0}, 1)], - ) - assert (out[0].ok, out[0].error) == (False, "Crashed (exit 0)") - assert out[1].ok is True diff --git a/code_runner/code_runner/tests/test_starter.py b/code_runner/code_runner/tests/test_starter.py deleted file mode 100644 index 7e47004..0000000 --- a/code_runner/code_runner/tests/test_starter.py +++ /dev/null @@ -1,78 +0,0 @@ -import pytest - -from code_runner._model import RunnerError -from code_runner._starter import Entry, extract_starter, parse_starter - -PY_CLASS = """class Solution: - def twoSum(self, nums: list[int], target: int) -> list[int]: -""" -PY_COMMENTED = """# Definition for singly-linked list. -# class ListNode: -# def __init__(self, val=0, next=None): -class Solution: - def solve(self, grid: Dict[str, Tuple[int, int]], k: int = 3, *rest) -> int: - pass -""" -PY_FUNC = "def add(a: int, b: int = 2) -> int:\n pass\n" -JS_VAR = """/** - * @param {number[]} nums - */ -var twoSum = function(nums, target) { - -};""" -JS_FUNC = "function isPalindrome(x, y = 3) {\n}\n" -JS_ARROW = "const f = (a, b) => {\n}\n" -JS_CLASS = "class Solution {\n constructor() {}\n solve(nums, ...rest) {\n }\n}\n" -TS_FUNC = "function twoSum(nums: number[], target: number): number[] {\n};" -TS_ARROW = "const f = (a: Map, b?: number): number => {\n}\n" - - -@pytest.mark.parametrize( - ("lang", "text", "want"), - [ - ("python3", PY_CLASS, Entry("Solution", "twoSum", ("nums", "target"))), - ("python3", PY_COMMENTED, Entry("Solution", "solve", ("grid", "k"))), - ("python3", PY_FUNC, Entry(None, "add", ("a", "b"))), - ("javascript", JS_VAR, Entry(None, "twoSum", ("nums", "target"))), - ("javascript", JS_FUNC, Entry(None, "isPalindrome", ("x", "y"))), - ("javascript", JS_ARROW, Entry(None, "f", ("a", "b"))), - ("javascript", JS_CLASS, Entry("Solution", "solve", ("nums",))), - ("typescript", TS_FUNC, Entry(None, "twoSum", ("nums", "target"))), - ("typescript", TS_ARROW, Entry(None, "f", ("a", "b"))), - ], -) -def test_parse_starter(lang: str, text: str, want: Entry) -> None: - assert parse_starter(lang, text) == want - - -@pytest.mark.parametrize( - ("lang", "text"), - [ - ("python3", "x = 1\n"), - ("python3", "class Solution:\n pass\n"), - ("javascript", "// nothing here\n"), - ("javascript", "class Solution {\n constructor() {}\n}\n"), - ("typescript", ""), - ], -) -def test_starter_without_entry_is_refused(lang: str, text: str) -> None: - with pytest.raises(RunnerError, match="entry point"): - parse_starter(lang, text) - - -def test_extract_starter_takes_the_fenced_block() -> None: - statement = ( - "# 1. Two Sum (Easy)\n\nbody\n\n## Starter code (python3)\n```\n" - + PY_CLASS - + "```\n\n## Site hints\nnope\n" - ) - assert extract_starter(statement) == PY_CLASS - - -def test_extract_starter_without_section_is_empty() -> None: - assert extract_starter("# 1. Two Sum\n\nno starter\n") == "" - - -def test_extract_starter_unfenced_runs_to_next_section() -> None: - text = "intro\n## Starter code (javascript)\nvar f = function(a) {};\n## Hints\nx\n" - assert extract_starter(text) == "var f = function(a) {};\n" diff --git a/code_runner/install.sh b/code_runner/install.sh deleted file mode 100755 index 5670699..0000000 --- a/code_runner/install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# Editable user install so /usr/bin/python3 (the tutor server, the CLI) can -# `import code_runner`. Stdlib only; idempotent. -set -euo pipefail - -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly REPO_DIR -readonly SYSTEM_PYTHON="/usr/bin/python3" - -"$SYSTEM_PYTHON" -m pip install --user --no-deps --break-system-packages -e "$REPO_DIR" -cd / -"$SYSTEM_PYTHON" -c 'import code_runner; print(code_runner.__file__)' diff --git a/code_runner/pyproject.toml b/code_runner/pyproject.toml deleted file mode 100644 index 1eb1da3..0000000 --- a/code_runner/pyproject.toml +++ /dev/null @@ -1,198 +0,0 @@ -[build-system] -requires = ["setuptools>=61"] -build-backend = "setuptools.build_meta" - -[project] -name = "code_runner" -version = "0.1.1" -description = "Run a learner solution (Python, JavaScript, TypeScript) against a problem's examples in a bubblewrap sandbox" -requires-python = ">=3.14" -# Stdlib only on purpose: the tutor server imports it from the system python. -dependencies = [] - -[tool.setuptools.packages.find] -include = ["code_runner*"] - -[tool.setuptools.package-data] -code_runner = ["py.typed", "harness/*"] - -[tool.ruff] -target-version = "py314" -include = ["*.py", "**/*.py"] -exclude = [".git", ".venv", "__pycache__", "build", "dist", ".eggs"] - -[tool.ruff.lint] -select = ["ALL"] -ignore = [ - "CPY001", - "D100", - "D101", - "D102", - "D103", - "D104", - "D105", - "D107", - "D203", - "D213", - "COM812", - "ISC001", - "S603", -] -fixable = ["ALL"] -unfixable = [] - -[tool.ruff.lint.per-file-ignores] -"scripts/*.py" = ["INP001"] -# The sandbox-side harness is a program of its own: it reads its job from files. -# The sandbox mounts its own private tmpfs at /tmp. -"**/_sandbox.py" = ["S108"] -"**/harness/*.py" = ["INP001", "T201", "S102", "BLE001", "PTH123"] -# Tests embed the learner programs they run as source strings. -"**/tests/**/*.py" = ["ARG", "D", "E501", "PLC0415", "PLR2004", "S101", "SLF001"] -"**/test_*.py" = ["ARG", "D", "E501", "PLC0415", "PLR2004", "S101", "SLF001"] - -[tool.ruff.lint.pylint] -max-args = 6 - -[tool.ruff.lint.pydocstyle] -convention = "google" - -[tool.ruff.lint.isort] -force-single-line = false -force-sort-within-sections = true -known-first-party = ["code_runner"] - -[tool.ruff.lint.flake8-quotes] -docstring-quotes = "double" -inline-quotes = "double" - -[tool.ruff.lint.flake8-tidy-imports] -ban-relative-imports = "all" - -[tool.ruff.format] -quote-style = "double" -indent-style = "space" -skip-magic-trailing-comma = false -line-ending = "auto" -docstring-code-format = true - -[tool.mypy] -python_version = "3.14" -strict = true -warn_return_any = true -warn_unused_configs = true -disallow_untyped_defs = true -disallow_incomplete_defs = true -check_untyped_defs = true -disallow_untyped_decorators = true -no_implicit_optional = true -warn_redundant_casts = true -warn_unused_ignores = true -warn_no_return = true -warn_unreachable = true -disallow_any_unimported = true -disallow_any_explicit = false -disallow_any_generics = true -disallow_subclassing_any = true -strict_equality = true -extra_checks = true -ignore_missing_imports = true -show_error_codes = true -color_output = true -exclude = [".venv/"] - -[tool.pylint.main] -# The repo root on the path, so pylint resolves the package's own imports the -# same way pytest does instead of reporting import-error for them. -init-hook = "import sys; sys.path.insert(0, 'code_runner')" -analyse-fallback-blocks = true -persistent = true -jobs = 0 -py-version = "3.14" -ignore = [".venv", "__pycache__"] -# Test suites intentionally use patterns (protected-access, fixture-arg -# shadowing) that don't apply to source code; ruff's per-file-ignores carry -# the equivalent exemptions, so tests stay linted -- by ruff. This is the same -# source-only pylint scope wake-alarm / diet-guard / testsAndMisc use. It has -# to be `ignore-paths`, not `ignore = ["tests"]`: `ignore` only matches the -# basename of what pylint is *given*, and pre-commit hands it explicit file -# paths, so a directory name in `ignore` never fires under the hook. -# harness/ runs inside the sandbox and must exec the solution it is given. -ignore-paths = ["^(.*/)?tests/.*$", "^(.*/)?harness/.*$"] -ignore-patterns = [".*\\.pyi$"] - -[tool.pylint.messages_control] -enable = "all" -disable = [ - "missing-module-docstring", - "missing-class-docstring", - "missing-function-docstring", - "use-implicit-booleaness-not-comparison-to-zero", -] - -[tool.pylint.design] -max-args = 6 -min-public-methods = 0 -max-module-lines = 1000 -max-attributes = 10 - -[tool.pylint.typecheck] -generated-members = [ - ".*\\.assert_called_once_with", - ".*\\.assert_called_once", - ".*\\.assert_called", - ".*\\.assert_not_called", - ".*\\.assert_any_call", - ".*\\.call_args", - ".*\\.call_args_list", - ".*\\.call_count", -] - -[tool.bandit] -exclude_dirs = ["tests", ".venv"] - -[tool.pytest.ini_options] -markers = ["sandbox: needs bubblewrap with unprivileged user namespaces (skipped otherwise)"] -testpaths = ["code_runner/tests"] -python_files = ["test_*.py", "*_test.py"] -python_classes = ["Test*"] -python_functions = ["test_*"] -addopts = [ - "-v", - "--strict-markers", - "--strict-config", - "-ra", - "--cov=code_runner", - "--cov-branch", - "--cov-report=term-missing", - "--cov-report=lcov", -] -filterwarnings = [ - "error", - "ignore::DeprecationWarning", - "default::pytest.PytestUnraisableExceptionWarning", -] - -[tool.coverage.run] -source = ["code_runner"] -branch = true -omit = [ - # Runs inside the sandbox in another process; behaviour is tested through run(). - "*/harness/*", - "*/__pycache__/*", - "*/tests/*", - "*/.venv/*", -] - -[tool.coverage.report] -fail_under = 100 -show_missing = true -skip_covered = false -exclude_lines = [ - "pragma: no cover", - "raise NotImplementedError", - "raise AssertionError", - "if TYPE_CHECKING:", - 'if __name__ == "__main__":', -] -partial_branches = ["pragma: no branch"] diff --git a/code_runner/requirements.txt b/code_runner/requirements.txt deleted file mode 100644 index d69c161..0000000 --- a/code_runner/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Exact pins, enforced by the shared dependency-freshness gate: a `>=` -# floor is satisfied by anything newer, so it tells a checker nothing -# about whether this repo is actually current. -bandit[toml]==1.9.4 -mypy==2.4.1 -pre-commit==4.7.0 -pylint==4.1.3 -pytest==9.2.0 -pytest-cov==7.1.0 -ruff==0.17.3