diff --git a/.github/workflows/code-runner-precommit.yml b/.github/workflows/code-runner-precommit.yml new file mode 100644 index 0000000..f427d98 --- /dev/null +++ b/.github/workflows/code-runner-precommit.yml @@ -0,0 +1,73 @@ +name: code-runner pre-commit + +on: + push: + branches: [main] + # The workflow's own file counts as code_runner: editing only this file would + # otherwise trigger nothing, so a fix to the job itself could never prove + # itself. + paths: ['code_runner/**', '.github/workflows/code-runner-precommit.yml'] + pull_request: + branches: [main] + paths: ['code_runner/**', '.github/workflows/code-runner-precommit.yml'] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + # Only pre-commit itself. Every linting hook runs in its own isolated + # env -- ruff, mypy, pylint, bandit and codespell each come from their + # own repo+rev, and the local hooks need nothing beyond bash and + # python3 -- so installing the full requirements.txt spent 9s a run on + # packages no pre-commit-stage hook imports. Verified by running all + # three configs green in a venv holding pre-commit and nothing else. + # The pin is read out of requirements.txt rather than repeated here, so + # it cannot drift from what the dependency-freshness gate governs. + - name: Install pre-commit + run: | + pip install "$(grep '^pre-commit==' code_runner/requirements.txt)" + + # Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on + # hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three + # package jobs shared one key, raced to overwrite it with hook sets built + # from four different configs, and editing a per-package config did not + # invalidate it. The cache was therefore silently wrong rather than + # missing, and the hooks reinstalled from scratch on every run (38-49s). + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: >- + pre-commit-code-runner-${{ env.pythonLocation }}-${{ + hashFiles('code_runner/.pre-commit-config.yaml') }} + # Prefix fallback, or this fix would evaporate on the very + # event that motivated it: bumping a hook rev changes the + # exact key, and a total miss means reinstalling every hook + # again. pre-commit's store is keyed internally by hook + # repo+rev, so a superseded entry is ignored disk rather + # than a wrong hook, and cache still re-saves under the new + # exact key after a prefix-only hit. + restore-keys: pre-commit-code-runner-${{ env.pythonLocation }}- + # mypy re-analyses the stdlib and every third-party stub from scratch + # without this, which measured 10s of the hook step in CI against 0.25s + # warm locally -- a 19MB cache dir does the whole difference. Keyed on + # the config, not the sources: mypy's own cache metadata records the + # interpreter, the flag set and a hash per source file, so it re-checks + # what changed and cannot pass stale code. Deliberately NOT keyed on + # github.sha -- that re-saves 19MB x3 every commit and would evict the + # 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps + # the expensive part, the dependency analysis, which is what costs 10s. + - uses: actions/cache@v4 + with: + path: .mypy_cache + key: >- + mypy-code-runner-${{ env.pythonLocation }}-${{ + hashFiles('code_runner/.pre-commit-config.yaml') }} + restore-keys: mypy-code-runner-${{ env.pythonLocation }}- + - name: Run the hooks + run: >- + pre-commit run --config code_runner/.pre-commit-config.yaml + --all-files --show-diff-on-failure diff --git a/.github/workflows/code-runner-tests.yml b/.github/workflows/code-runner-tests.yml new file mode 100644 index 0000000..f21688d --- /dev/null +++ b/.github/workflows/code-runner-tests.yml @@ -0,0 +1,34 @@ +name: code-runner tests + +on: + push: + branches: [main] + # The workflow's own file counts as code_runner: editing only this file would + # otherwise trigger nothing, so a fix to the job itself could never prove + # itself. + paths: ['code_runner/**', '.github/workflows/code-runner-tests.yml'] + pull_request: + branches: [main] + paths: ['code_runner/**', '.github/workflows/code-runner-tests.yml'] + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: code_runner + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + cache: pip + cache-dependency-path: code_runner/requirements.txt + + - name: Install dependencies + run: pip install -r requirements.txt && pip install --no-deps -e . + + - name: Run tests with coverage + run: python -m pytest -v diff --git a/code_runner/.pre-commit-config.yaml b/code_runner/.pre-commit-config.yaml new file mode 100644 index 0000000..08be4d5 --- /dev/null +++ b/code_runner/.pre-commit-config.yaml @@ -0,0 +1,143 @@ +# Pre-commit Configuration for code_runner +# Install: pre-commit install && pre-commit install --hook-type pre-push +# Run: pre-commit run --all-files +# Update: pre-commit autoupdate + +# NOTE: this subproject lives inside the ~/src/utils monorepo, whose git root is +# one level up. pre-commit always runs hooks from the git root, so every path +# below is written relative to THAT, not to this directory, and the top-level +# `files` key keeps these hooks off the sibling subprojects. CI invokes this +# config exactly the same way: +# pre-commit run --config code_runner/.pre-commit-config.yaml --all-files +files: ^code_runner/ + +default_language_version: + python: python3 +default_stages: [pre-commit] +fail_fast: false + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-added-large-files + args: [--maxkb=2000] + - id: check-merge-conflict + - id: detect-private-key + - id: debug-statements + - id: name-tests-test + args: [--pytest-test-first] + - id: check-ast + - id: mixed-line-ending + args: [--fix=lf] + - id: requirements-txt-fixer + + - repo: local + hooks: + - id: file-length-cap + name: file length <= 250 lines + entry: bash scripts/check_file_length.sh + language: system + pass_filenames: true + - id: subproject-scope + name: subproject scope regex still matches + entry: scripts/check_subproject_scope.sh code_runner + language: script + # always_run so the top-level `files` filter cannot skip the one hook + # whose job is to notice that filter has stopped matching anything. + always_run: true + pass_filenames: false + - id: no-noqa + name: Block noqa comments + entry: '(?i)#\s*(noqa|type:\s*ignore)' + language: pygrep + types: [python] + - id: no-ruff-noqa + name: Block ruff noqa file-level comments + entry: '(?i)#\s*ruff:\s*noqa' + language: pygrep + types: [python] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.2 + hooks: + - id: ruff + args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix, --show-fixes] + types_or: [python, pyi] + - id: ruff-format + types_or: [python, pyi] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.13.0 + hooks: + - id: mypy + args: + - --ignore-missing-imports + - --no-error-summary + - --disable-error-code=no-untyped-def + - --disable-error-code=no-untyped-call + - --disable-error-code=var-annotated + - --disable-error-code=no-any-unimported + - --disable-error-code=type-arg + - --disable-error-code=no-any-return + - --disable-error-code=misc + - --disable-error-code=unused-ignore + - --disable-error-code=unreachable + - --disable-error-code=assignment + - --disable-error-code=no-redef + - --disable-error-code=attr-defined + - --disable-error-code=arg-type + - --disable-error-code=union-attr + - --disable-error-code=call-overload + - --disable-error-code=return-value + - --disable-error-code=redundant-cast + - --disable-error-code=empty-body + - --disable-error-code=list-item + + - repo: https://github.com/pylint-dev/pylint + rev: v4.0.8 + hooks: + - id: pylint + args: + - --rcfile=code_runner/pyproject.toml + - --fail-under=10 + - --jobs=4 + additional_dependencies: + - pytest + + - repo: https://github.com/PyCQA/bandit + rev: 1.9.4 + hooks: + - id: bandit + args: + - -c + - code_runner/pyproject.toml + - --severity-level=high + - --confidence-level=medium + - --skip=B113 + additional_dependencies: ["bandit[toml]"] + exclude: ^(code_runner/code_runner/tests/|.*test.*\.py$) + + - repo: local + hooks: + - id: pytest-coverage + name: pytest with coverage enforcement + entry: scripts/run_subproject_tests.sh code_runner + language: script + types: [python] + pass_filenames: false + require_serial: true + stages: [pre-push] + + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + args: + - --skip=*.json,*.lock,.git,__pycache__,.venv diff --git a/code_runner/README.md b/code_runner/README.md new file mode 100644 index 0000000..86145a3 --- /dev/null +++ b/code_runner/README.md @@ -0,0 +1,12 @@ +# code_runner + +Runs a learner's solution (Python 3, JavaScript, TypeScript) against a problem statement's examples inside bubblewrap: no network, read-only `/usr`, scratch dir, CPU/memory/process/file/time limits. + +```python +import code_runner as cr + +statement = ... # markdown from ~/.claude/skills/leetcode/fetch_problem.py +results = cr.run("python3", code, cr.extract_starter(statement), cr.parse_examples(statement)) +``` + +`run(lang, code, starter, examples) -> list[CaseResult]` with `n, input, expected, actual, ok, stdout, error, ms`. One sandbox process per example. TypeScript runs through node's type stripping (no type check, no enums). Needs `bwrap`, `prlimit`, `/usr/bin/python3`, node >= 22.18. Install: `./install.sh`. diff --git a/code_runner/code_runner/__init__.py b/code_runner/code_runner/__init__.py new file mode 100644 index 0000000..5dfd6dc --- /dev/null +++ b/code_runner/code_runner/__init__.py @@ -0,0 +1,20 @@ +"""Run a learner's solution against a problem's examples in a bubblewrap sandbox.""" + +from __future__ import annotations + +from code_runner._examples import parse_examples +from code_runner._model import CaseResult, Example, Limits, RunnerError +from code_runner._run import run +from code_runner._starter import Entry, extract_starter, parse_starter + +__all__ = [ + "CaseResult", + "Entry", + "Example", + "Limits", + "RunnerError", + "extract_starter", + "parse_examples", + "parse_starter", + "run", +] diff --git a/code_runner/code_runner/_compare.py b/code_runner/code_runner/_compare.py new file mode 100644 index 0000000..52a3be3 --- /dev/null +++ b/code_runner/code_runner/_compare.py @@ -0,0 +1,66 @@ +"""Output comparison: exact JSON equality, float tolerance, order rules.""" + +from __future__ import annotations + +import json +import math + +TOLERANCE = 1e-5 # LeetCode accepts floating answers within 1e-5 + + +def _as_number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return value + + +def _key(value: object) -> str: + return json.dumps(value, sort_keys=True, ensure_ascii=False) + + +def _deep_sorted(value: object) -> object: + if isinstance(value, list): + return sorted((_deep_sorted(item) for item in value), key=_key) + return value + + +def _equal(expected: object, actual: object) -> bool: + want, got = _as_number(expected), _as_number(actual) + if want is not None and got is not None: + return math.isclose(want, got, rel_tol=TOLERANCE, abs_tol=TOLERANCE) + if isinstance(expected, list) and isinstance(actual, list): + return len(expected) == len(actual) and all( + _equal(e, a) for e, a in zip(expected, actual, strict=True) + ) + if isinstance(expected, dict) and isinstance(actual, dict): + return expected.keys() == actual.keys() and all( + _equal(expected[k], actual[k]) for k in expected + ) + return type(expected) is type(actual) and expected == actual + + +def _top_sorted(value: object) -> object: + if isinstance(value, list): + return sorted(value, key=_key) + return value + + +def matches(expected: object, actual: object, order: str) -> bool: + """Whether ``actual`` is an accepted answer for ``expected``. + + Args: + expected: the statement's answer (JSON value). + actual: what the solution returned (JSON value). + order: "exact", "any" (list order free) or "deep" (nested order free too). + + Raises: + ValueError: ``order`` is none of those. + """ + if order == "exact": + return _equal(expected, actual) + if order == "any": + return _equal(_top_sorted(expected), _top_sorted(actual)) + if order == "deep": + return _equal(_deep_sorted(expected), _deep_sorted(actual)) + msg = f"unknown order {order!r}" + raise ValueError(msg) diff --git a/code_runner/code_runner/_examples.py b/code_runner/code_runner/_examples.py new file mode 100644 index 0000000..9e90209 --- /dev/null +++ b/code_runner/code_runner/_examples.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import ast +import json +import logging +import re +from typing import Final + +from code_runner._model import Example, Order +from code_runner._split import split_top_level + +_logger: Final = logging.getLogger(__name__) +_LABEL: Final = re.compile(r"^(Input|Output)\s*:\s*(.*)$") +_PAIR: Final = re.compile(r"^([A-Za-z_]\w*)\s*=\s*(.+)$", re.DOTALL) +_ANY_ORDER: Final = re.compile(r"any order|order of the output", re.IGNORECASE) +_DEEP_ORDER: Final = re.compile( + r"order of the (?:triplets|quadruplets|subsets|groups|pairs|elements)", + re.IGNORECASE, +) + + +def _value(text: str) -> object: + try: + return json.loads(text) + except ValueError: + try: + return ast.literal_eval(text) + except (ValueError, SyntaxError) as exc: + raise ValueError(text[:40]) from exc + + +def _arguments(text: str) -> dict[str, object]: + args: dict[str, object] = {} + for part in split_top_level(text): + pair = _PAIR.match(part) + if pair is None: + msg = f"not a name = value pair: {part[:40]!r}" + raise ValueError(msg) + args[pair.group(1)] = _value(pair.group(2)) + return args + + +def _labels(body: str) -> list[tuple[str, str]]: + cleaned = ( + line.replace("**", "").replace("`", "").strip() for line in body.splitlines() + ) + matched = (_LABEL.match(line) for line in cleaned) + return [(m.group(1), m.group(2).strip()) for m in matched if m is not None] + + +def _order(body: str) -> Order: + if _DEEP_ORDER.search(body): + return "deep" + return "any" if _ANY_ORDER.search(body) else "exact" + + +def parse_examples(statement: str) -> list[Example]: + body = re.split(r"^## ", statement, maxsplit=1, flags=re.MULTILINE)[0] + order = _order(body) + found: list[Example] = [] + pending: str | None = None + number = 0 + for label, text in _labels(body): + if label == "Input": + pending = text + continue + if pending is None: + continue + number += 1 + try: + found.append(Example(number, _arguments(pending), _value(text), order)) + except ValueError as exc: + _logger.warning("example %d skipped, not runnable: %s", number, exc) + pending = None + return found diff --git a/code_runner/code_runner/_languages.py b/code_runner/code_runner/_languages.py new file mode 100644 index 0000000..36be09d --- /dev/null +++ b/code_runner/code_runner/_languages.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from dataclasses import dataclass +from importlib import resources +import json +import os +from pathlib import Path +import shutil +from typing import TYPE_CHECKING + +from code_runner._model import RunnerError +from code_runner._sandbox import WORK + +if TYPE_CHECKING: + from code_runner._starter import Entry + +MEGABYTE = 1024 * 1024 +_NODE_FLAGS = ( + "--permission", + f"--allow-fs-read={WORK}", + f"--allow-fs-write={WORK}", + "--max-old-space-size=128", +) + + +@dataclass(frozen=True) +class Language: + name: str + solution_file: str + harness_file: str + program: str + binary_env: str + binary_default: str + flags: tuple[str, ...] + memory_bytes: int + + def resolve(self) -> Path: + for candidate in ( + os.environ.get(self.binary_env), + self.binary_default, + shutil.which(self.program), + ): + if candidate and Path(candidate).is_file(): + return Path(candidate).resolve() + msg = f"{self.program} not found (set {self.binary_env})" + raise RunnerError(msg) + + def command(self, binary: Path) -> list[str]: + return [str(binary), *self.flags, f"{WORK}/{self.harness_file}", WORK] + + def ro_binds(self, binary: Path) -> tuple[str, ...]: + if binary.is_relative_to("/usr"): + return () + return (str(binary.parent.parent),) + + def prepare( + self, scratch: Path, code: str, entry: Entry, args: list[object] + ) -> None: + harness = resources.files("code_runner").joinpath("harness", self.harness_file) + (scratch / self.harness_file).write_text(harness.read_text(encoding="utf-8")) + text = code + if self.name == "typescript": + text += f"\nmodule.exports = {{ __entry: {entry.cls or entry.name} }};\n" + (scratch / self.solution_file).write_text(text, encoding="utf-8") + spec = { + "cls": entry.cls, + "fn": entry.name, + "args": args, + "typescript": self.name == "typescript", + } + (scratch / "input.json").write_text(json.dumps(spec), encoding="utf-8") + + +PYTHON = Language( + "python3", + "solution.py", + "harness.py", + "python3", + "CODE_RUNNER_PYTHON", + "/usr/bin/python3", + ("-I", "-B"), + 512 * MEGABYTE, +) +JAVASCRIPT = Language( + "javascript", + "solution.js", + "harness.js", + "node", + "CODE_RUNNER_NODE", + "/usr/bin/node", + _NODE_FLAGS, + 2048 * MEGABYTE, +) +TYPESCRIPT = Language( + "typescript", + "solution.cts", + "harness.js", + "node", + "CODE_RUNNER_NODE", + "/usr/bin/node", + _NODE_FLAGS, + 2048 * MEGABYTE, +) +_BY_NAME = { + "python3": PYTHON, + "python": PYTHON, + "javascript": JAVASCRIPT, + "js": JAVASCRIPT, + "typescript": TYPESCRIPT, + "ts": TYPESCRIPT, +} + + +def get_language(name: str) -> Language: + try: + return _BY_NAME[name.lower()] + except KeyError: + msg = f"unsupported language {name!r}; use python3, javascript or typescript" + raise RunnerError(msg) from None diff --git a/code_runner/code_runner/_model.py b/code_runner/code_runner/_model.py new file mode 100644 index 0000000..dbcad4e --- /dev/null +++ b/code_runner/code_runner/_model.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +# "any": list order is free; "deep": the order inside its items is free too. +Order = Literal["exact", "any", "deep"] +MAX_CAPTURE = 64 * 1024 + + +class RunnerError(Exception): + pass + + +@dataclass(frozen=True) +class Example: + n: int + args: dict[str, object] + expected: object + order: Order = "exact" + + +@dataclass(frozen=True) +class CaseResult: + n: int + input: dict[str, object] + expected: object + actual: object + ok: bool + stdout: str = "" + error: str | None = None + ms: float = 0.0 + + +@dataclass(frozen=True) +class Limits: + timeout_s: float = 2.0 + nproc: int = 64 + fsize_bytes: int = 1024 * 1024 + nofile: int = 256 diff --git a/code_runner/code_runner/_run.py b/code_runner/code_runner/_run.py new file mode 100644 index 0000000..525c5df --- /dev/null +++ b/code_runner/code_runner/_run.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +import json +import os +from pathlib import Path +import stat +import tempfile + +from code_runner._compare import matches +from code_runner._languages import Language, get_language +from code_runner._model import MAX_CAPTURE, CaseResult, Example, Limits +from code_runner._sandbox import BoxResult, execute +from code_runner._starter import Entry, parse_starter + +Box = Callable[[list[str], Path, Limits, int, tuple[str, ...]], BoxResult] +_OUT_OF_MEMORY = ("out of memory", "allocation failed") + + +def _positional(example: Example, entry: Entry) -> list[object]: + if set(example.args) == set(entry.params): + return [example.args[name] for name in entry.params] + if len(example.args) == len(entry.params): + return list(example.args.values()) + given, wanted = len(example.args), len(entry.params) + msg = f"ArgumentError: example gives {given} arguments, {entry.name} takes {wanted}" + raise ValueError(msg) + + +def _read_result(scratch: Path) -> dict[str, object] | None: + # The scratch dir is the solution's: never follow a link it planted there. + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + try: + fd = os.open(scratch / "result.json", flags) + except OSError: + return None + with os.fdopen(fd, "rb") as handle: + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + return None + raw = handle.read(MAX_CAPTURE * 16) + try: + payload = json.loads(raw) + except ValueError: + return None + return payload if isinstance(payload, dict) and "ok" in payload else None + + +def _crash(box: BoxResult) -> str: + low = box.stderr.lower() + if any(marker in low for marker in _OUT_OF_MEMORY): + return "MemoryLimitExceeded" + return f"Crashed (exit {box.returncode})" + + +def _outcome(scratch: Path, box: BoxResult) -> tuple[object, str | None, float]: + if box.timed_out: + return None, "TimeLimitExceeded", box.elapsed_ms + payload = _read_result(scratch) + if payload is None: + return None, _crash(box), box.elapsed_ms + ms = payload.get("ms") + took = float(ms) if isinstance(ms, int | float) else box.elapsed_ms + if payload["ok"] is True: + return payload.get("value"), None, took + return None, str(payload.get("error", "Error")), took + + +@dataclass(frozen=True) +class _Job: + language: Language + binary: Path + code: str + entry: Entry + limits: Limits + box: Box + + +def _case(job: _Job, example: Example) -> CaseResult: + try: + args = _positional(example, job.entry) + except ValueError as exc: + return CaseResult( + example.n, example.args, example.expected, None, ok=False, error=str(exc) + ) + language = job.language + with tempfile.TemporaryDirectory(prefix="code-runner-") as tmp: + scratch = Path(tmp) + language.prepare(scratch, job.code, job.entry, args) + done = job.box( + language.command(job.binary), + scratch, + job.limits, + language.memory_bytes, + language.ro_binds(job.binary), + ) + actual, error, ms = _outcome(scratch, done) + ok = error is None and matches(example.expected, actual, example.order) + return CaseResult( + example.n, example.args, example.expected, actual, ok, done.stdout, error, ms + ) + + +def run( + lang: str, + code: str, + starter: str, + examples: Sequence[Example], + *, + limits: Limits | None = None, + box: Box = execute, +) -> list[CaseResult]: + language = get_language(lang) + entry = parse_starter(language.name, starter) + job = _Job(language, language.resolve(), code, entry, limits or Limits(), box) + return [_case(job, example) for example in examples] diff --git a/code_runner/code_runner/_sandbox.py b/code_runner/code_runner/_sandbox.py new file mode 100644 index 0000000..60c51b5 --- /dev/null +++ b/code_runner/code_runner/_sandbox.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from dataclasses import dataclass +import functools +import math +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import time +from typing import IO + +from code_runner._model import MAX_CAPTURE, Limits, RunnerError + +WORK = "/work" +PRLIMIT = "/usr/bin/prlimit" +_ROOT_LINKS = ("bin", "sbin", "lib", "lib64") + + +@dataclass(frozen=True) +class BoxResult: + returncode: int + timed_out: bool + stdout: str + stderr: str + elapsed_ms: float + + +def _root_links() -> list[str]: + args: list[str] = [] + for name in _ROOT_LINKS: + path = Path("/") / name + if path.is_symlink(): + args += ["--symlink", str(path.readlink()), str(path)] + elif path.is_dir(): + args += ["--ro-bind", str(path), str(path)] + return args + + +def _base_argv() -> list[str]: + return [ + "bwrap", + "--unshare-all", # no --share-net: no network at all, loopback included + "--die-with-parent", + "--new-session", + "--clearenv", + "--ro-bind", + "/usr", + "/usr", + *_root_links(), + "--proc", + "/proc", + "--dev", + "/dev", + "--tmpfs", + "/tmp", + ] + + +def bwrap_argv( + command: list[str], + scratch: Path, + limits: Limits, + memory_bytes: int, + ro_binds: tuple[str, ...] = (), +) -> list[str]: + argv = _base_argv() + for bind in ro_binds: + argv += ["--ro-bind", bind, bind] + argv += ["--bind", str(scratch), WORK, "--remount-ro", "/", "--chdir", WORK] + for name, value in ( + ("HOME", WORK), + ("PATH", "/usr/bin"), + ("LANG", "C.UTF-8"), + ("PYTHONIOENCODING", "utf-8"), + ): + argv += ["--setenv", name, value] + cpu = math.ceil(limits.timeout_s) + 1 + return [ + *argv, + "--", + PRLIMIT, + f"--as={memory_bytes}", + f"--nproc={limits.nproc}", + f"--fsize={limits.fsize_bytes}", + f"--nofile={limits.nofile}", + f"--cpu={cpu}", + "--core=0", + "--", + *command, + ] + + +@functools.cache +def check_sandbox() -> None: + if not Path(PRLIMIT).is_file(): + msg = f"{PRLIMIT} is missing (util-linux)" + raise RunnerError(msg) + try: + probe = subprocess.run( + [*_base_argv(), "/usr/bin/true"], + capture_output=True, + check=False, + timeout=20, + ) + except OSError as exc: + msg = f"bubblewrap (bwrap) cannot start: {exc}" + raise RunnerError(msg) from exc + if probe.returncode != 0: + detail = probe.stderr.decode(errors="replace").strip() + msg = f"bubblewrap cannot create the sandbox: {detail}" + raise RunnerError(msg) + + +def _read(handle: IO[bytes]) -> str: + handle.seek(0) + return handle.read(MAX_CAPTURE).decode("utf-8", errors="replace") + + +def execute( + command: list[str], + scratch: Path, + limits: Limits, + memory_bytes: int, + ro_binds: tuple[str, ...] = (), +) -> BoxResult: + check_sandbox() + argv = bwrap_argv(command, scratch, limits, memory_bytes, ro_binds) + with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: + started = time.monotonic() + with subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=out, + stderr=err, + env={"PATH": "/usr/bin:/bin"}, + start_new_session=True, + ) as proc: + timed_out = False + try: + proc.wait(timeout=limits.timeout_s) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(proc.pid, signal.SIGKILL) + elapsed = (time.monotonic() - started) * 1000 + return BoxResult(proc.returncode, timed_out, _read(out), _read(err), elapsed) diff --git a/code_runner/code_runner/_split.py b/code_runner/code_runner/_split.py new file mode 100644 index 0000000..cacd2ee --- /dev/null +++ b/code_runner/code_runner/_split.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +_OPEN = "([{<" +_CLOSE = ")]}>" + + +def split_top_level(text: str) -> list[str]: + parts: list[str] = [] + depth = 0 + quote = "" + escaped = False + start = 0 + for i, ch in enumerate(text): + if escaped: + escaped = False + elif quote: + escaped = ch == "\\" + quote = "" if ch == quote else quote + elif ch in "\"'": + quote = ch + elif ch in _OPEN: + depth += 1 + elif ch in _CLOSE and not (ch == ">" and text[i - 1] in "=-"): + depth = max(depth - 1, 0) + elif ch == "," and depth == 0: + parts.append(text[start:i]) + start = i + 1 + parts.append(text[start:]) + return [part.strip() for part in parts] diff --git a/code_runner/code_runner/_starter.py b/code_runner/code_runner/_starter.py new file mode 100644 index 0000000..fa74ff0 --- /dev/null +++ b/code_runner/code_runner/_starter.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from dataclasses import dataclass +import re + +from code_runner._model import RunnerError +from code_runner._split import split_top_level + +_IDENT = r"[A-Za-z_$][\w$]*" +_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) +_LINE_COMMENT = re.compile(r"^\s*(?://|#).*$", re.MULTILINE) +_PY_CLASS = re.compile(r"^class\s+(\w+)", re.MULTILINE) +_PY_DEF = re.compile(r"\bdef\s+(?!__)(\w+)\s*\(") +_PY_FUNC = re.compile(r"^def\s+(\w+)\s*\(", re.MULTILINE) +_JS_CLASS = re.compile(rf"^\s*class\s+({_IDENT})", re.MULTILINE) +_JS_METHOD = re.compile( + rf"^\s+(?:async\s+)?(?!constructor\b)({_IDENT})\s*\(", re.MULTILINE +) +_JS_ASSIGNED = re.compile( + rf"\b(?:var|let|const)\s+({_IDENT})\s*(?::[^=]+)?=\s*(?:async\s+)?(?:function\b\s*\w*\s*)?\(" +) +_JS_DECLARED = re.compile(rf"\bfunction\s+({_IDENT})\s*\(") +_NOT_METHODS = {"if", "for", "while", "switch", "catch", "return", "function"} + + +@dataclass(frozen=True) +class Entry: + cls: str | None + name: str + params: tuple[str, ...] + + +def _no_entry(lang: str) -> RunnerError: + return RunnerError(f"no entry point found in the {lang} starter code") + + +def _balanced(text: str, open_at: int) -> str: + depth = 0 + for i in range(open_at, len(text)): + ch = text[i] + if ch in "([{" or (ch == "<" and depth > 0): + depth += 1 + elif ch in ")]}" or (ch == ">" and text[i - 1] != "=" and depth > 1): + depth -= 1 + if depth == 0: + return text[open_at + 1 : i] + return text[open_at + 1 :] + + +def _params(code: str, open_at: int) -> tuple[str, ...]: + names: list[str] = [] + for part in split_top_level(_balanced(code, open_at)): + found = re.match(_IDENT, part.strip()) + if found: + names.append(found.group()) + return tuple(names) + + +def _python(text: str) -> Entry | None: + code = _LINE_COMMENT.sub("", text) + owner = _PY_CLASS.search(code) + if owner is None: + found = _PY_FUNC.search(code) + return found and Entry(None, found.group(1), _params(code, found.end() - 1)) + found = _PY_DEF.search(code, owner.end()) + if found is None: + return None + params = _params(code, found.end() - 1) + return Entry( + owner.group(1), found.group(1), tuple(p for p in params if p != "self") + ) + + +def _javascript(text: str) -> Entry | None: + code = _LINE_COMMENT.sub("", _BLOCK_COMMENT.sub("", text)) + owner = _JS_CLASS.search(code) + if owner is not None: + for method in _JS_METHOD.finditer(code, owner.end()): + if method.group(1) not in _NOT_METHODS: + return Entry( + owner.group(1), method.group(1), _params(code, method.end() - 1) + ) + return None + found = _JS_ASSIGNED.search(code) or _JS_DECLARED.search(code) + return found and Entry(None, found.group(1), _params(code, found.end() - 1)) + + +def parse_starter(lang: str, text: str) -> Entry: + entry = _python(text) if lang == "python3" else _javascript(text) + if entry is None: + raise _no_entry(lang) + return entry + + +def extract_starter(statement: str) -> str: + lines = statement.splitlines(keepends=True) + heads = [i for i, line in enumerate(lines) if line.startswith("## Starter code")] + if not heads: + return "" + body = lines[heads[0] + 1 :] + if body and body[0].startswith("```"): + body = body[1:] + end = next( + (i for i, line in enumerate(body) if line.startswith("```")), len(body) + ) + return "".join(body[:end]) + end = next((i for i, line in enumerate(body) if line.startswith("## ")), len(body)) + return "".join(body[:end]) diff --git a/code_runner/code_runner/harness/harness.js b/code_runner/code_runner/harness/harness.js new file mode 100644 index 0000000..ab975af --- /dev/null +++ b/code_runner/code_runner/harness/harness.js @@ -0,0 +1,45 @@ +"use strict"; +// Runs inside the sandbox: call the solution once, write result.json. +const fs = require("node:fs"); +const path = require("node:path"); + +const root = process.argv[2]; +const spec = JSON.parse(fs.readFileSync(path.join(root, "input.json"), "utf8")); +const IDENT = /^[A-Za-z_$][\w$]*$/; + +function describe(e) { + const name = e && typeof e.name === "string" ? e.name : "Error"; + const message = e && e.message !== undefined ? e.message : String(e); + return `${name}: ${message}`.slice(0, 300); +} + +function writeResult(payload) { + // Unique name + rename: a symlink planted at result.json is replaced, never followed. + const unique = path.join(root, `.result-${process.pid}-${Math.random().toString(36).slice(2)}`); + fs.writeFileSync(unique, JSON.stringify(payload), { flag: "wx" }); + fs.renameSync(unique, path.join(root, "result.json")); +} + +function load(target) { + if (!IDENT.test(target)) throw new TypeError("bad entry name"); + if (spec.typescript) return require(path.join(root, "solution.cts")).__entry; + const source = fs.readFileSync(path.join(root, "solution.js"), "utf8"); + return new Function(`${source}\n;return ${target};`)(); +} + +function call() { + const entry = load(spec.cls || spec.fn); + if (spec.cls) return new entry()[spec.fn](...spec.args); + return entry(...spec.args); +} + +const started = process.hrtime.bigint(); +let payload; +try { + const value = call(); + payload = { ok: true, value: value === undefined ? null : JSON.parse(JSON.stringify(value)) }; +} catch (e) { + payload = { ok: false, error: describe(e) }; +} +payload.ms = Number(process.hrtime.bigint() - started) / 1e6; +writeResult(payload); diff --git a/code_runner/code_runner/harness/harness.py b/code_runner/code_runner/harness/harness.py new file mode 100644 index 0000000..f04c8a8 --- /dev/null +++ b/code_runner/code_runner/harness/harness.py @@ -0,0 +1,69 @@ +"""Runs inside the sandbox: call the solution once, write result.json.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import sys +import tempfile +import time +from typing import Any + +PRELUDE = ( + "from typing import *\n" + "import bisect, collections, functools, heapq, itertools, math\n" + "import operator, re, string\n" +) +MAX_MESSAGE = 300 + + +def describe(exc: BaseException) -> str: + return f"{type(exc).__name__}: {exc}"[:MAX_MESSAGE] + + +def write_result(root: Path, payload: dict[str, object]) -> None: + # Unique name + rename: a symlink the solution planted at result.json is + # replaced by this file, never followed. + handle, name = tempfile.mkstemp(dir=root, prefix=".result-") + with os.fdopen(handle, "w", encoding="utf-8") as out: + json.dump(payload, out) + Path(name).replace(root / "result.json") + + +def call(root: Path, spec: dict[str, Any]) -> object: + namespace: dict[str, Any] = {"__name__": "solution"} + exec(PRELUDE, namespace) + source = (root / "solution.py").read_text(encoding="utf-8") + exec(compile(source, "solution.py", "exec"), namespace) + owner, name = spec["cls"], str(spec["fn"]) + if owner is None: + target = namespace.get(name) + else: + cls = namespace.get(str(owner)) + target = getattr(cls() if cls else None, name, None) + if target is None: + message = f"name {owner or name!r} is not defined" + raise NameError(message) + return target(*spec["args"]) + + +def main(directory: str) -> None: + root = Path(directory) + spec = json.loads((root / "input.json").read_text(encoding="utf-8")) + started = time.perf_counter() + try: + value = call(root, spec) + payload: dict[str, object] = { + "ok": True, + "value": json.loads(json.dumps(value)), + } + except BaseException as exc: + payload = {"ok": False, "error": describe(exc)} + payload["ms"] = (time.perf_counter() - started) * 1000 + sys.stdout.flush() + write_result(root, payload) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/code_runner/code_runner/py.typed b/code_runner/code_runner/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/code_runner/code_runner/tests/__init__.py b/code_runner/code_runner/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/code_runner/code_runner/tests/conftest.py b/code_runner/code_runner/tests/conftest.py new file mode 100644 index 0000000..d55c3eb --- /dev/null +++ b/code_runner/code_runner/tests/conftest.py @@ -0,0 +1,70 @@ +"""Shared fixtures: real-sandbox tests are skipped where bubblewrap cannot run.""" + +from __future__ import annotations + +import shutil + +import pytest + +from code_runner._model import Example, RunnerError +from code_runner._sandbox import check_sandbox + + +def _bwrap_works() -> bool: + try: + check_sandbox() + except RunnerError: + return False + return True + + +needs_sandbox = pytest.mark.skipif(not _bwrap_works(), reason="bwrap/userns unusable") +needs_node = pytest.mark.skipif( + not _bwrap_works() or not shutil.which("node"), reason="node or bwrap missing" +) + +TWO_SUM_STARTER = { + "python3": "class Solution:\n def twoSum(self, nums: list[int], target: int) -> list[int]:\n", + "javascript": "var twoSum = function(nums, target) {\n\n};", + "typescript": "function twoSum(nums: number[], target: number): number[] {\n};", +} +TWO_SUM_CODE = { + "python3": ( + "class Solution:\n" + " def twoSum(self, nums, target):\n" + " seen = {}\n" + " for i, n in enumerate(nums):\n" + " if target - n in seen:\n" + " return [i, seen[target - n]]\n" + " seen[n] = i\n" + ), + "javascript": ( + "var twoSum = function(nums, target) {\n" + " const seen = new Map();\n" + " for (let i = 0; i < nums.length; i++) {\n" + " if (seen.has(target - nums[i])) return [i, seen.get(target - nums[i])];\n" + " seen.set(nums[i], i);\n" + " }\n" + "};" + ), + "typescript": ( + "function twoSum(nums: number[], target: number): number[] {\n" + " const seen = new Map();\n" + " for (let i = 0; i < nums.length; i++) {\n" + " const j = seen.get(target - nums[i]);\n" + " if (j !== undefined) return [i, j];\n" + " seen.set(nums[i], i);\n" + " }\n" + " return [];\n" + "}" + ), +} + + +@pytest.fixture +def two_sum_examples() -> list[Example]: + return [ + Example(1, {"nums": [2, 7, 11, 15], "target": 9}, [0, 1], "any"), + Example(2, {"nums": [3, 2, 4], "target": 6}, [1, 2], "any"), + Example(3, {"nums": [3, 3], "target": 6}, [0, 1], "any"), + ] diff --git a/code_runner/code_runner/tests/test_compare.py b/code_runner/code_runner/tests/test_compare.py new file mode 100644 index 0000000..bb9d959 --- /dev/null +++ b/code_runner/code_runner/tests/test_compare.py @@ -0,0 +1,42 @@ +import pytest + +from code_runner._compare import matches + + +@pytest.mark.parametrize( + ("expected", "actual", "order", "want"), + [ + ([0, 1], [0, 1], "exact", True), + ([0, 1], [1, 0], "exact", False), + ([0, 1], [1, 0], "any", True), + ([0, 1], [1, 0, 2], "any", False), + ([[1, 2], [3]], [[3], [1, 2]], "any", True), + ([[1, 2], [3]], [[3], [2, 1]], "any", False), + ([[1, 2], [3]], [[3], [2, 1]], "deep", True), + ([[1, 2], [3]], [[3], [2, 9]], "deep", False), + (3, 3.0, "exact", True), + (0.1 + 0.2, 0.3, "exact", True), + (2.0, 2.001, "exact", False), + (True, 1, "exact", False), + (1, True, "exact", False), + (True, True, "exact", True), + (None, None, "exact", True), + (None, 0, "exact", False), + ("ab", "ab", "exact", True), + ("ab", "ba", "exact", False), + ("1", 1, "exact", False), + ({"a": [1]}, {"a": [1]}, "exact", True), + ({"a": 1}, {"b": 1}, "exact", False), + ({"a": 1}, {"a": 2}, "exact", False), + (5, 5, "any", True), + ([1], 1, "any", False), + ([1.0, 2.0], [2.0, 1.0000001], "any", True), + ], +) +def test_matches(expected: object, actual: object, order: str, *, want: bool) -> None: + assert matches(expected, actual, order) is want + + +def test_unknown_order_is_refused() -> None: + with pytest.raises(ValueError, match="order"): + matches(1, 1, "sideways") diff --git a/code_runner/code_runner/tests/test_examples.py b/code_runner/code_runner/tests/test_examples.py new file mode 100644 index 0000000..f7622e5 --- /dev/null +++ b/code_runner/code_runner/tests/test_examples.py @@ -0,0 +1,106 @@ +from code_runner._examples import parse_examples +from code_runner._model import Example + +TWO_SUM = """# 1. Two Sum (Easy) +Tags: Array, Hash Table + +Return indices. + +You can return the answer in any order. + +**Example 1:** + +**Input:** `nums = [2,7,11,15], target = 9` + +**Output:** `[0,1]` + +**Explanation:** 2 + 7. + +**Example 2:** + +**Input:** `nums = [3,2,4], target = 6` + +**Output:** `[1,2]` + +## Starter code (python3) +``` +class Solution: + def twoSum(self, nums, target): +``` +""" + +FENCED = """# 5. Pal + +Output order does not matter. + +**Example 1:** +``` +Input: s = "a,b", flag = true +Output: "b" +Explanation: Output: not an example +``` +**Example 2:** +``` +Input: s = 'x', flag = false +Output: None +``` +""" + +THREE_SUM = """# 15. 3Sum +Notice that the order of the output and the order of the triplets does not matter. +**Example 1:** +**Input:** nums = [-1,0,1,2,-1,-4] +**Output:** [[-1,-1,2],[-1,0,1]] +""" + +SKIPPED = """# 2. Add Two Numbers +**Example 1:** +**Input:** l1 = 1->2 +**Output:** [1,2] +**Example 2:** +**Input:** a = 1, b = 2 +**Output:** 3.00000 +**Example 3:** +**Input:** a = 1 +**Output:** 1->2 +**Example 4:** +**Input:** a +**Output:** 1 +""" + + +def test_two_sum_examples_are_typed_and_ordered_any() -> None: + found = parse_examples(TWO_SUM) + assert found == [ + Example(1, {"nums": [2, 7, 11, 15], "target": 9}, [0, 1], "any"), + Example(2, {"nums": [3, 2, 4], "target": 6}, [1, 2], "any"), + ] + + +def test_fenced_examples_and_python_literals() -> None: + found = parse_examples(FENCED) + assert [e.args for e in found] == [ + {"s": "a,b", "flag": True}, + {"s": "x", "flag": False}, + ] + assert [e.expected for e in found] == ["b", None] + assert {e.order for e in found} == {"exact"} + + +def test_triplets_sentence_means_deep_order() -> None: + assert parse_examples(THREE_SUM)[0].order == "deep" + + +def test_unparseable_examples_are_skipped_but_numbering_is_kept() -> None: + found = parse_examples(SKIPPED) + assert found == [Example(2, {"a": 1, "b": 2}, 3.0, "exact")] + + +def test_output_without_input_and_empty_statement() -> None: + assert parse_examples("**Output:** 1\n") == [] + assert parse_examples("") == [] + + +def test_starter_section_is_never_scanned() -> None: + text = "# t\n## Starter code (python3)\n```\nInput: a = 1\nOutput: 2\n```\n" + assert parse_examples(text) == [] diff --git a/code_runner/code_runner/tests/test_internals.py b/code_runner/code_runner/tests/test_internals.py new file mode 100644 index 0000000..33bb58f --- /dev/null +++ b/code_runner/code_runner/tests/test_internals.py @@ -0,0 +1,192 @@ +"""Branches that a healthy sandbox never reaches: crashes, bad results, no bwrap.""" + +from __future__ import annotations + +import dataclasses +import os +from pathlib import Path +import subprocess +from typing import TYPE_CHECKING + +import pytest + +from code_runner import Example, Limits, RunnerError, run +from code_runner._examples import parse_examples +from code_runner._languages import JAVASCRIPT, PYTHON, get_language +from code_runner._run import Box, _read_result +from code_runner._sandbox import BoxResult, bwrap_argv, check_sandbox +from code_runner._starter import parse_starter + +if TYPE_CHECKING: + from collections.abc import Iterator + +STARTER = "def f(a):\n" +EXAMPLE = Example(1, {"a": 1}, 1) + + +@pytest.fixture(autouse=True) +def _fresh_sandbox_check() -> Iterator[None]: + check_sandbox.cache_clear() + yield + check_sandbox.cache_clear() + + +def _fake(result: BoxResult, files: dict[str, str] | None = None) -> Box: + def box( + command: list[str], + scratch: Path, + limits: Limits, + memory: int, + binds: tuple[str, ...], + ) -> BoxResult: + for name, text in (files or {}).items(): + (scratch / name).write_text(text) + return result + + return box + + +def _box(**kw: object) -> BoxResult: + fields = { + "returncode": 0, + "timed_out": False, + "stdout": "", + "stderr": "", + "elapsed_ms": 7.0, + } + return BoxResult(**(fields | kw)) + + +def _run(box: Box) -> object: + return run("python3", "", STARTER, [EXAMPLE], box=box)[0] + + +def test_crash_without_result_names_the_exit_code() -> None: + result = _run(_fake(_box(returncode=139))) + assert (result.ok, result.actual, result.error, result.ms) == ( + False, + None, + "Crashed (exit 139)", + 7.0, + ) + + +def test_out_of_memory_crash_is_named() -> None: + stderr = "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory" + assert ( + _run(_fake(_box(returncode=134, stderr=stderr))).error == "MemoryLimitExceeded" + ) + + +def test_timeout_wins_over_a_result_file() -> None: + files = {"result.json": '{"ok": true, "value": 1}'} + result = _run(_fake(_box(timed_out=True, elapsed_ms=1000.0), files)) + assert (result.error, result.ms) == ("TimeLimitExceeded", 1000.0) + + +def test_result_file_variants() -> None: + good = _run( + _fake(_box(stdout="hi"), {"result.json": '{"ok": true, "value": 1, "ms": 2.5}'}) + ) + assert (good.ok, good.ms, good.stdout) == (True, 2.5, "hi") + no_ms = _run(_fake(_box(), {"result.json": '{"ok": true, "value": 1}'})) + assert no_ms.ms == 7.0 + failed = _run(_fake(_box(), {"result.json": '{"ok": false}'})) + assert failed.error == "Error" + named = _run( + _fake(_box(), {"result.json": '{"ok": false, "error": "KeyError: x"}'}) + ) + assert named.error == "KeyError: x" + + +def test_unreadable_result_files_are_a_crash(tmp_path: Path) -> None: + secret = tmp_path / "secret" + secret.write_text('{"ok": true, "value": "stolen"}') + (tmp_path / "result.json").symlink_to(secret) + assert _read_result(tmp_path) is None + (tmp_path / "result.json").unlink() + os.mkfifo(tmp_path / "result.json") + assert _read_result(tmp_path) is None + (tmp_path / "result.json").unlink() + for text in ("not json", "[1]", '{"no": "ok"}'): + (tmp_path / "result.json").write_text(text) + assert _read_result(tmp_path) is None + assert _read_result(tmp_path / "missing") is None + + +def test_binary_resolution(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + ghost = dataclasses.replace( + PYTHON, binary_default="/nope", program="no-such-program-xyz" + ) + monkeypatch.delenv(ghost.binary_env, raising=False) + with pytest.raises(RunnerError, match="no-such-program-xyz not found"): + ghost.resolve() + tool = tmp_path / "bin" / "python3" + tool.parent.mkdir() + tool.write_text("#!/bin/sh\n") + monkeypatch.setenv(ghost.binary_env, str(tool)) + assert ghost.resolve() == tool + assert ghost.ro_binds(tool) == (str(tool.parent.parent),) + assert PYTHON.ro_binds(Path("/usr/bin/python3")) == () + assert get_language("TS").name == "typescript" + + +def _has(argv: list[str], seq: list[str]) -> bool: + return any(argv[i : i + len(seq)] == seq for i in range(len(argv))) + + +def test_argv_binds_links_and_limits( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr("code_runner._sandbox._ROOT_LINKS", ("usr", "no-such-root-dir")) + argv = bwrap_argv(["prog"], tmp_path, Limits(timeout_s=1.2), 123, ("/opt/x",)) + assert _has(argv, ["--ro-bind", "/usr", "/usr"]) + assert _has(argv, ["--ro-bind", "/opt/x", "/opt/x"]) + assert _has(argv, ["--bind", str(tmp_path), "/work"]) + assert "--share-net" not in argv + assert {"--unshare-all", "--die-with-parent", "--clearenv", "--remount-ro"} <= set( + argv + ) + assert "--as=123" in argv + assert "--cpu=3" in argv + assert argv[-1] == "prog" + + +def test_sandbox_preflight_failures(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("code_runner._sandbox.PRLIMIT", "/no/such/prlimit") + with pytest.raises(RunnerError, match="prlimit"): + check_sandbox() + monkeypatch.undo() + + def missing(*args: object, **kwargs: object) -> None: + message = "bwrap" + raise FileNotFoundError(message) + + monkeypatch.setattr(subprocess, "run", missing) + with pytest.raises(RunnerError, match="cannot start"): + check_sandbox() + + def refused(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: + return subprocess.CompletedProcess(args, 1, b"", b"bwrap: No permissions\n") + + monkeypatch.setattr(subprocess, "run", refused) + with pytest.raises(RunnerError, match="No permissions"): + check_sandbox() + + +def test_statement_and_starter_edge_cases() -> None: + escaped = parse_examples('**Input:** s = "a\\"b,c", t = 1\n**Output:** "x"\n') + assert escaped[0].args == {"s": 'a"b,c', "t": 1} + assert parse_starter("python3", "def f(a, b").params == ("a", "b") + skipped = "class S {\n if (x) {\n solve(a) {\n }\n}\n" + assert parse_starter("javascript", skipped).name == "solve" + + +def test_typescript_class_entry_is_exported(tmp_path: Path) -> None: + entry = parse_starter( + "typescript", "class Solution {\n solve(a: number): number {\n }\n}\n" + ) + get_language("typescript").prepare(tmp_path, "class Solution {}", entry, [1]) + assert "__entry: Solution" in (tmp_path / "solution.cts").read_text() + JAVASCRIPT.prepare(tmp_path, "x", entry, [1]) + assert (tmp_path / "solution.js").read_text() == "x" diff --git a/code_runner/code_runner/tests/test_isolation.py b/code_runner/code_runner/tests/test_isolation.py new file mode 100644 index 0000000..bb877e5 --- /dev/null +++ b/code_runner/code_runner/tests/test_isolation.py @@ -0,0 +1,206 @@ +"""Escape attempts: each one runs hostile code in the real sandbox and must fail.""" + +from __future__ import annotations + +from pathlib import Path +import socket +import subprocess +import time +from typing import Any, cast + +import pytest + +from code_runner import CaseResult, Example, Limits, run +from code_runner.tests.conftest import needs_node, needs_sandbox + +pytestmark = needs_sandbox +STARTER = "def f(a):\n" +FAST = Limits(timeout_s=1.0) + + +def _one(code: str, lang: str = "python3", limits: Limits = FAST) -> CaseResult: + starter = {"python3": STARTER, "javascript": "var f = function(a) {};"}[lang] + return run(lang, code, starter, [Example(1, {"a": 0}, 1)], limits=limits)[0] + + +def _pgrep(pattern: str) -> int: + done = subprocess.run( + ["/usr/bin/pgrep", "-fc", pattern], capture_output=True, text=True, check=False + ) + return int(done.stdout.strip() or 0) + + +def test_no_network_not_even_host_loopback() -> None: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + listener.settimeout(0) + port = listener.getsockname()[1] + code = ( + "import socket\n" + "def f(a):\n" + " out = []\n" + f" for host, port in [('127.0.0.1', {port}), ('1.1.1.1', 53), ('::1', {port})]:\n" + " try:\n" + " socket.create_connection((host, port), timeout=1).close()\n" + " out.append('connected')\n" + " except OSError:\n" + " out.append('blocked')\n" + " return out\n" + ) + result = _one(code) + with pytest.raises(BlockingIOError): + listener.accept() + assert result.actual == ["blocked", "blocked", "blocked"] + + +@needs_node +def test_no_network_from_node() -> None: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + port = listener.getsockname()[1] + code = ( + "var f = function(a) {\n" + " const net = process.getBuiltinModule('node:net');\n" + " let seen = 'pending';\n" + f" const s = net.connect({port}, '127.0.0.1');\n" + " s.on('connect', () => { seen = 'connected'; });\n" + " s.on('error', () => {});\n" + " return seen;\n" + "};" + ) + result = _one(code, "javascript") + listener.settimeout(0.2) + with pytest.raises(TimeoutError): + listener.accept() + assert result.actual == "pending" + + +def test_writes_outside_scratch_never_reach_the_host(tmp_path: Path) -> None: + marker = tmp_path / "marker" + code = ( + "def f(a):\n" + " out = []\n" + f" for path in ['/usr/escape', '/etc/escape', '/escape', {str(marker)!r}]:\n" + " try:\n" + " open(path, 'w').write('x')\n" + " out.append('wrote')\n" + " except OSError:\n" + " out.append('denied')\n" + " return out\n" + ) + result = _one(code) + assert result.actual == ["denied", "denied", "denied", "denied"] + assert not marker.exists() + assert not Path("/usr/escape").exists() + + +def test_host_files_and_environment_are_invisible( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + secret = tmp_path / "secret.txt" + secret.write_text("hunter2") + monkeypatch.setenv("CODE_RUNNER_SECRET", "hunter2") + code = ( + "import os\n" + "def f(a):\n" + f" return [os.path.exists({str(secret)!r}), os.path.exists({str(Path.home())!r}),\n" + " 'CODE_RUNNER_SECRET' in os.environ, sorted(os.environ), os.getpid() < 20,\n" + " os.path.exists('/proc/1/environ') and 'hunter2' in open('/proc/1/environ').read()]\n" + ) + values = cast("list[Any]", _one(code).actual) + exists, home, env_leak, names, own_pidns, proc_leak = values + assert (exists, home, env_leak, own_pidns, proc_leak) == ( + False, + False, + False, + True, + False, + ) + assert set(names) <= {"HOME", "LANG", "PATH", "PWD", "PYTHONIOENCODING"} + + +def test_symlink_in_scratch_cannot_exfiltrate_a_host_file(tmp_path: Path) -> None: + secret = tmp_path / "secret.txt" + secret.write_text("hunter2-secret-contents") + code = ( + "import os\n" + "def f(a):\n" + f" os.symlink({str(secret)!r}, '/work/result.json')\n" + " return 7\n" + ) + result = _one(code) + assert "hunter2" not in repr(result) + assert result.actual == 7 # the harness replaced the link, never followed it + + +def test_fork_bomb_is_contained_and_leaves_nothing_behind() -> None: + before = _pgrep("python3 -I /work/harness.py") + code = ( + "import os, subprocess\n" + "def f(a):\n" + " subprocess.Popen(['sleep', '97'])\n" + " while True:\n" + " os.fork()\n" + ) + started = time.monotonic() + result = _one(code) + assert time.monotonic() - started < 6 + assert result.ok is False + assert result.error is not None + assert result.error.startswith(("TimeLimitExceeded", "BlockingIOError")) + time.sleep(0.3) + assert _pgrep("python3 -I /work/harness.py") == before + assert _pgrep("^sleep 97$") == 0 + + +def test_memory_bomb_is_a_visible_error() -> None: + result = _one("def f(a):\n return len(bytearray(8 * 1024**3))\n") + assert result.ok is False + assert result.error is not None + assert result.error.startswith("MemoryError") + + +@needs_node +def test_node_heap_bomb_is_a_visible_error() -> None: + code = "var f = function(a) { const x = []; for (;;) x.push(new Array(1e6).fill(1)); };" + result = _one(code, "javascript", Limits(timeout_s=5.0)) + assert result.ok is False + assert result.error in {"MemoryLimitExceeded", "RangeError: Invalid array length"} + + +def test_output_flood_is_capped() -> None: + code = "def f(a):\n while True:\n print('x' * 1000)\n" + started = time.monotonic() + result = _one(code) + assert time.monotonic() - started < 6 + assert result.ok is False + assert len(result.stdout) <= 64 * 1024 + + +def test_disk_fill_is_capped_by_the_file_size_limit() -> None: + code = ( + "def f(a):\n" + " with open('/work/big', 'wb') as fh:\n" + " while True:\n" + " fh.write(b'x' * 65536)\n" + ) + result = _one(code) + assert result.ok is False + assert result.error is not None + assert "OSError" in result.error + + +def test_timeout_kills_every_process_the_solution_started() -> None: + code = ( + "import subprocess\n" + "def f(a):\n" + " subprocess.Popen(['sleep', '97'])\n" + " while True:\n" + " pass\n" + ) + result = _one(code) + assert result.error == "TimeLimitExceeded" + time.sleep(0.3) + assert _pgrep("^sleep 97$") == 0 diff --git a/code_runner/code_runner/tests/test_run.py b/code_runner/code_runner/tests/test_run.py new file mode 100644 index 0000000..c3d6ffd --- /dev/null +++ b/code_runner/code_runner/tests/test_run.py @@ -0,0 +1,207 @@ +"""End to end through the real sandbox: every language, every kind of outcome.""" + +from __future__ import annotations + +import pytest + +from code_runner import CaseResult, Example, Limits, RunnerError, run +from code_runner.tests.conftest import ( + TWO_SUM_CODE, + TWO_SUM_STARTER, + needs_node, + needs_sandbox, +) + +LANGS = [ + pytest.param("python3", marks=needs_sandbox), + pytest.param("javascript", marks=needs_node), + pytest.param("typescript", marks=needs_node), +] +FAST = Limits(timeout_s=1.0) + + +def _two_sum(lang: str, code: str | None, examples: list[Example]) -> list[CaseResult]: + return run(lang, code or TWO_SUM_CODE[lang], TWO_SUM_STARTER[lang], examples) + + +@pytest.mark.parametrize("lang", LANGS) +def test_correct_solution_passes_every_example( + lang: str, two_sum_examples: list[Example] +) -> None: + results = _two_sum(lang, None, two_sum_examples) + assert [r.ok for r in results] == [True, True, True] + assert [r.n for r in results] == [1, 2, 3] + first = results[0] + assert first.input == {"nums": [2, 7, 11, 15], "target": 9} + assert first.expected == [0, 1] + assert sorted(first.actual) == [0, 1] # any order: [1, 0] is fine + assert first.error is None + assert first.ms >= 0 + + +WRONG = { + "python3": "class Solution:\n def twoSum(self, nums, target):\n return [0, 0]\n", + "javascript": "var twoSum = function(nums, target) { return [0, 0]; };", + "typescript": "function twoSum(nums: number[], target: number): number[] { return [0, 0]; }", +} + + +@pytest.mark.parametrize("lang", LANGS) +def test_wrong_answer_is_reported_with_actual_value( + lang: str, two_sum_examples: list[Example] +) -> None: + results = _two_sum(lang, WRONG[lang], two_sum_examples) + assert [r.ok for r in results] == [False, False, False] + assert results[0].actual == [0, 0] + assert results[0].error is None + + +THROWS = { + "python3": "class Solution:\n def twoSum(self, nums, target):\n return nums[99]\n", + "javascript": "var twoSum = function(nums, target) { throw new RangeError('boom'); };", + "typescript": "function twoSum(nums: number[], target: number): number[] { throw new RangeError('boom'); }", +} + + +@pytest.mark.parametrize("lang", LANGS) +def test_exception_becomes_an_error_per_case( + lang: str, two_sum_examples: list[Example] +) -> None: + results = _two_sum(lang, THROWS[lang], two_sum_examples[:1]) + only = results[0] + assert only.ok is False + assert only.actual is None + assert only.error is not None + assert only.error.split(":")[0] in {"IndexError", "RangeError"} + + +BROKEN = { + "python3": "class Solution:\n def twoSum(self, nums, target) return 1\n", + "javascript": "var twoSum = function(nums, target) { return ; ;; }}", + "typescript": "function twoSum(nums: number[], target: number): number[] { return [ }", +} + + +@pytest.mark.parametrize("lang", LANGS) +def test_syntax_error_is_reported(lang: str, two_sum_examples: list[Example]) -> None: + results = _two_sum(lang, BROKEN[lang], two_sum_examples[:2]) + assert [r.ok for r in results] == [False, False] + assert all(r.error and r.error.startswith("SyntaxError") for r in results) + + +PRINTS = { + "python3": "class Solution:\n def twoSum(self, nums, target):\n print('hi', target)\n return [0, 1]\n", + "javascript": "var twoSum = function(nums, target) { console.log('hi', target); return [0, 1]; };", + "typescript": "function twoSum(nums: number[], target: number): number[] { console.log('hi', target); return [0, 1]; }", +} + + +@pytest.mark.parametrize("lang", LANGS) +def test_prints_land_in_stdout_per_case( + lang: str, two_sum_examples: list[Example] +) -> None: + results = _two_sum(lang, PRINTS[lang], two_sum_examples[:2]) + assert results[0].stdout.strip() == "hi 9" + assert results[1].stdout.strip() == "hi 6" + + +@pytest.mark.parametrize("lang", LANGS) +def test_missing_entry_point_in_code_is_a_case_error( + lang: str, two_sum_examples: list[Example] +) -> None: + code = { + "python3": "x = 1\n", + "javascript": "var other = 1;", + "typescript": "const other = 1;", + } + results = _two_sum(lang, code[lang], two_sum_examples[:1]) + assert results[0].ok is False + assert results[0].error is not None + assert "twoSum" in results[0].error or "Solution" in results[0].error + + +def test_class_style_javascript_and_function_style_python() -> None: + js = run( + "javascript", + "class Solution { solve(a, b) { return a + b; } }", + "class Solution {\n solve(a, b) {\n }\n}", + [Example(1, {"a": 1, "b": 2}, 3)], + ) + py = run( + "python3", + "def add(a, b):\n return a + b\n", + "def add(a: int, b: int) -> int:\n", + [Example(1, {"a": 1, "b": 2}, 3)], + ) + assert js[0].ok is True + assert py[0].ok is True + + +def test_python_prelude_and_aliases() -> None: + code = "class Solution:\n def f(self, xs: List[int]) -> int:\n return collections.Counter(xs)[1]\n" + starter = "class Solution:\n def f(self, xs: List[int]) -> int:\n" + out = run("python", code, starter, [Example(1, {"xs": [1, 1, 2]}, 2)]) + assert out[0].ok is True + + +def test_positional_fallback_when_statement_names_differ() -> None: + out = run( + "python3", + "def add(a, b):\n return a - b\n", + "def add(a, b):\n", + [Example(1, {"x": 5, "y": 2}, 3)], + ) + assert out[0].ok is True + + +def test_argument_count_mismatch_is_a_case_error_without_running() -> None: + out = run( + "python3", + "def add(a, b):\n return a\n", + "def add(a, b):\n", + [Example(1, {"only": 5}, 3)], + ) + assert out[0].ok is False + assert out[0].error is not None + assert out[0].error.startswith("ArgumentError") + + +def test_non_json_result_is_a_case_error() -> None: + out = run( + "python3", + "def f(a):\n return {1, 2}\n", + "def f(a):\n", + [Example(1, {"a": 1}, [1, 2])], + ) + assert out[0].error is not None + assert out[0].error.startswith("TypeError") + + +def test_deep_order_and_float_tolerance() -> None: + out = run( + "python3", + "def f(a):\n return [[2, 1], [0, -1]]\n", + "def f(a):\n", + [Example(1, {"a": 0}, [[-1, 0], [1, 2]], "deep")], + ) + assert out[0].ok is True + + +def test_timeout_is_per_case() -> None: + out = run( + "python3", + "def f(a):\n while a:\n pass\n return 1\n", + "def f(a):\n", + [Example(1, {"a": 1}, 1), Example(2, {"a": 0}, 1)], + limits=FAST, + ) + assert out[0].error == "TimeLimitExceeded" + assert out[0].ok is False + assert out[0].ms >= 900 + assert out[1].ok is True + + +def test_unsupported_language_and_empty_examples() -> None: + with pytest.raises(RunnerError, match="language"): + run("cobol", "", "", []) + assert run("python3", "def f(a): pass\n", "def f(a):\n", []) == [] diff --git a/code_runner/code_runner/tests/test_starter.py b/code_runner/code_runner/tests/test_starter.py new file mode 100644 index 0000000..7e47004 --- /dev/null +++ b/code_runner/code_runner/tests/test_starter.py @@ -0,0 +1,78 @@ +import pytest + +from code_runner._model import RunnerError +from code_runner._starter import Entry, extract_starter, parse_starter + +PY_CLASS = """class Solution: + def twoSum(self, nums: list[int], target: int) -> list[int]: +""" +PY_COMMENTED = """# Definition for singly-linked list. +# class ListNode: +# def __init__(self, val=0, next=None): +class Solution: + def solve(self, grid: Dict[str, Tuple[int, int]], k: int = 3, *rest) -> int: + pass +""" +PY_FUNC = "def add(a: int, b: int = 2) -> int:\n pass\n" +JS_VAR = """/** + * @param {number[]} nums + */ +var twoSum = function(nums, target) { + +};""" +JS_FUNC = "function isPalindrome(x, y = 3) {\n}\n" +JS_ARROW = "const f = (a, b) => {\n}\n" +JS_CLASS = "class Solution {\n constructor() {}\n solve(nums, ...rest) {\n }\n}\n" +TS_FUNC = "function twoSum(nums: number[], target: number): number[] {\n};" +TS_ARROW = "const f = (a: Map, b?: number): number => {\n}\n" + + +@pytest.mark.parametrize( + ("lang", "text", "want"), + [ + ("python3", PY_CLASS, Entry("Solution", "twoSum", ("nums", "target"))), + ("python3", PY_COMMENTED, Entry("Solution", "solve", ("grid", "k"))), + ("python3", PY_FUNC, Entry(None, "add", ("a", "b"))), + ("javascript", JS_VAR, Entry(None, "twoSum", ("nums", "target"))), + ("javascript", JS_FUNC, Entry(None, "isPalindrome", ("x", "y"))), + ("javascript", JS_ARROW, Entry(None, "f", ("a", "b"))), + ("javascript", JS_CLASS, Entry("Solution", "solve", ("nums",))), + ("typescript", TS_FUNC, Entry(None, "twoSum", ("nums", "target"))), + ("typescript", TS_ARROW, Entry(None, "f", ("a", "b"))), + ], +) +def test_parse_starter(lang: str, text: str, want: Entry) -> None: + assert parse_starter(lang, text) == want + + +@pytest.mark.parametrize( + ("lang", "text"), + [ + ("python3", "x = 1\n"), + ("python3", "class Solution:\n pass\n"), + ("javascript", "// nothing here\n"), + ("javascript", "class Solution {\n constructor() {}\n}\n"), + ("typescript", ""), + ], +) +def test_starter_without_entry_is_refused(lang: str, text: str) -> None: + with pytest.raises(RunnerError, match="entry point"): + parse_starter(lang, text) + + +def test_extract_starter_takes_the_fenced_block() -> None: + statement = ( + "# 1. Two Sum (Easy)\n\nbody\n\n## Starter code (python3)\n```\n" + + PY_CLASS + + "```\n\n## Site hints\nnope\n" + ) + assert extract_starter(statement) == PY_CLASS + + +def test_extract_starter_without_section_is_empty() -> None: + assert extract_starter("# 1. Two Sum\n\nno starter\n") == "" + + +def test_extract_starter_unfenced_runs_to_next_section() -> None: + text = "intro\n## Starter code (javascript)\nvar f = function(a) {};\n## Hints\nx\n" + assert extract_starter(text) == "var f = function(a) {};\n" diff --git a/code_runner/install.sh b/code_runner/install.sh new file mode 100755 index 0000000..5670699 --- /dev/null +++ b/code_runner/install.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Editable user install so /usr/bin/python3 (the tutor server, the CLI) can +# `import code_runner`. Stdlib only; idempotent. +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_DIR +readonly SYSTEM_PYTHON="/usr/bin/python3" + +"$SYSTEM_PYTHON" -m pip install --user --no-deps --break-system-packages -e "$REPO_DIR" +cd / +"$SYSTEM_PYTHON" -c 'import code_runner; print(code_runner.__file__)' diff --git a/code_runner/pyproject.toml b/code_runner/pyproject.toml new file mode 100644 index 0000000..d77a4e4 --- /dev/null +++ b/code_runner/pyproject.toml @@ -0,0 +1,198 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "code_runner" +version = "0.1.0" +description = "Run a learner solution (Python, JavaScript, TypeScript) against a problem's examples in a bubblewrap sandbox" +requires-python = ">=3.14" +# Stdlib only on purpose: the tutor server imports it from the system python. +dependencies = [] + +[tool.setuptools.packages.find] +include = ["code_runner*"] + +[tool.setuptools.package-data] +code_runner = ["py.typed", "harness/*"] + +[tool.ruff] +target-version = "py314" +include = ["*.py", "**/*.py"] +exclude = [".git", ".venv", "__pycache__", "build", "dist", ".eggs"] + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "CPY001", + "D100", + "D101", + "D102", + "D103", + "D104", + "D105", + "D107", + "D203", + "D213", + "COM812", + "ISC001", + "S603", +] +fixable = ["ALL"] +unfixable = [] + +[tool.ruff.lint.per-file-ignores] +"scripts/*.py" = ["INP001"] +# The sandbox-side harness is a program of its own: it reads its job from files. +# The sandbox mounts its own private tmpfs at /tmp. +"**/_sandbox.py" = ["S108"] +"**/harness/*.py" = ["INP001", "T201", "S102", "BLE001", "PTH123"] +# Tests embed the learner programs they run as source strings. +"**/tests/**/*.py" = ["ARG", "D", "E501", "PLC0415", "PLR2004", "S101", "SLF001"] +"**/test_*.py" = ["ARG", "D", "E501", "PLC0415", "PLR2004", "S101", "SLF001"] + +[tool.ruff.lint.pylint] +max-args = 6 + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +force-single-line = false +force-sort-within-sections = true +known-first-party = ["code_runner"] + +[tool.ruff.lint.flake8-quotes] +docstring-quotes = "double" +inline-quotes = "double" + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +docstring-code-format = true + +[tool.mypy] +python_version = "3.14" +strict = true +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +disallow_any_unimported = true +disallow_any_explicit = false +disallow_any_generics = true +disallow_subclassing_any = true +strict_equality = true +extra_checks = true +ignore_missing_imports = true +show_error_codes = true +color_output = true +exclude = [".venv/"] + +[tool.pylint.main] +# The repo root on the path, so pylint resolves the package's own imports the +# same way pytest does instead of reporting import-error for them. +init-hook = "import sys; sys.path.insert(0, 'code_runner')" +analyse-fallback-blocks = true +persistent = true +jobs = 0 +py-version = "3.14" +ignore = [".venv", "__pycache__"] +# Test suites intentionally use patterns (protected-access, fixture-arg +# shadowing) that don't apply to source code; ruff's per-file-ignores carry +# the equivalent exemptions, so tests stay linted -- by ruff. This is the same +# source-only pylint scope wake-alarm / diet-guard / testsAndMisc use. It has +# to be `ignore-paths`, not `ignore = ["tests"]`: `ignore` only matches the +# basename of what pylint is *given*, and pre-commit hands it explicit file +# paths, so a directory name in `ignore` never fires under the hook. +# harness/ runs inside the sandbox and must exec the solution it is given. +ignore-paths = ["^(.*/)?tests/.*$", "^(.*/)?harness/.*$"] +ignore-patterns = [".*\\.pyi$"] + +[tool.pylint.messages_control] +enable = "all" +disable = [ + "missing-module-docstring", + "missing-class-docstring", + "missing-function-docstring", + "use-implicit-booleaness-not-comparison-to-zero", +] + +[tool.pylint.design] +max-args = 6 +min-public-methods = 0 +max-module-lines = 1000 +max-attributes = 10 + +[tool.pylint.typecheck] +generated-members = [ + ".*\\.assert_called_once_with", + ".*\\.assert_called_once", + ".*\\.assert_called", + ".*\\.assert_not_called", + ".*\\.assert_any_call", + ".*\\.call_args", + ".*\\.call_args_list", + ".*\\.call_count", +] + +[tool.bandit] +exclude_dirs = ["tests", ".venv"] + +[tool.pytest.ini_options] +markers = ["sandbox: needs bubblewrap with unprivileged user namespaces (skipped otherwise)"] +testpaths = ["code_runner/tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", + "--strict-config", + "-ra", + "--cov=code_runner", + "--cov-branch", + "--cov-report=term-missing", + "--cov-report=lcov", +] +filterwarnings = [ + "error", + "ignore::DeprecationWarning", + "default::pytest.PytestUnraisableExceptionWarning", +] + +[tool.coverage.run] +source = ["code_runner"] +branch = true +omit = [ + # Runs inside the sandbox in another process; behaviour is tested through run(). + "*/harness/*", + "*/__pycache__/*", + "*/tests/*", + "*/.venv/*", +] + +[tool.coverage.report] +fail_under = 100 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "raise NotImplementedError", + "raise AssertionError", + "if TYPE_CHECKING:", + 'if __name__ == "__main__":', +] +partial_branches = ["pragma: no branch"] diff --git a/code_runner/requirements.txt b/code_runner/requirements.txt new file mode 100644 index 0000000..d69c161 --- /dev/null +++ b/code_runner/requirements.txt @@ -0,0 +1,10 @@ +# Exact pins, enforced by the shared dependency-freshness gate: a `>=` +# floor is satisfied by anything newer, so it tells a checker nothing +# about whether this repo is actually current. +bandit[toml]==1.9.4 +mypy==2.4.1 +pre-commit==4.7.0 +pylint==4.1.3 +pytest==9.2.0 +pytest-cov==7.1.0 +ruff==0.17.3