diff --git a/rtdb_relay/README.md b/rtdb_relay/README.md new file mode 100644 index 0000000..6d92d3d --- /dev/null +++ b/rtdb_relay/README.md @@ -0,0 +1,90 @@ +# rtdb_relay + +A sealed request relay over Firebase RTDB: a phone app writes a request, a +worker on the PC answers it. No VPN, no open port: both sides talk only to +RTDB over HTTPS. Access needs the one Google account the RTDB rules allow, +and every value is sealed end to end with the shared crdt-sync passphrase. + +Extracted from `kuhyx/tutor` (`tutor/relay/`), whose phone app has used this +wire format since 2026-10. The tutor's contract vectors +(`rtdb_relay/tests/data/e2e_vectors.json`) pin the byte format; the tests +check that the package seals and opens exactly those bytes. + +Python 3.14+ (tested on 3.15.2). Install from a release tag: + +```toml +"rtdb-relay @ git+https://github.com/kuhyx/utils@rtdb-relay-v0.1.0#subdirectory=rtdb_relay" +``` + +## Wire format (under the app's root, e.g. `todoimpl/v1-e2e`) + +| Path | Who writes | Value | +|---|---|---| +| `requests/{rid}/req` | client | sealed `{kind, dev, created, sent_at, body}`; `created` clear | +| `requests/{rid}/claim` | worker | clear `{by, lease_until, attempt, phase, at}` (ms) | +| `requests/{rid}/result` | worker | sealed `{ok, kind, ...data}` or `{ok: false, kind, code, error, retry}`; `at` clear | +| `presence/workers/{worker}` | worker | sealed `{host, version}`; `beat` clear (server time) | +| `/e2e` | first unlock | the Argon2id key header (crdt_sync) | + +`rid` is a ULID (`[0-9A-HJKMNP-TV-Z]{26}`); any other key is never used in +a path. A sealed unit is `{"_e2e": , }`, the blob being crdt_sync's `E2eCipher` over compact JSON with the +unit's full path (`/`) as AAD, so a blob moved elsewhere does not +open. + +## Python API (the PC worker) + +```python +from crdt_sync import E2eCipher, E2eLayout +from rtdb_relay.layout import Layout, Rule, E2eLayoutError, parts_of +from rtdb_relay.seal import Sealer, SEALED +from rtdb_relay.rtdb import Rtdb, RtdbError, ConflictError, SERVER_TIME +from rtdb_relay.unlock import unlock +from rtdb_relay.server import RequestServer, RequestError +from rtdb_relay import presence, sse, mirror +``` + +- `Layout(root: str, clear: tuple[str, ...], units: dict[str, frozenset[str]])`: + the tree rules. `*` matches one segment; a `clear` pattern covers its + subtree; a unit maps to the clear fields it keeps. A write to a path with + no rule raises `E2eLayoutError` (fail closed). A relay app needs at least + `clear=("requests/*/claim",)` and units `requests/*/req: {"created"}`, + `requests/*/result: {"at"}`, `presence/workers/*: {"beat"}`. + `layout.classify(parts) -> Rule`, `layout.unit_path(path) -> str | None`. +- `Rtdb(database_url: str, token: Callable[[], Awaitable[str]], client: + httpx.AsyncClient | None = None, *, root: str, sealer: Sealer | None = None)`: + `get(path, params=None)`, `get_etag(path)`, `put(path, value)`, + `put_etag(path, value, if_match)`, `patch(path, dict)`, `delete(path, + if_match=None)`, `open_stream(path, params=None)`, `at(root) -> Rtdb` (raw, + unsealed), `aclose()`. Errors never contain the URL or the token. +- `await unlock(raw_rtdb, passphrase: str, layout: E2eLayout, params: + Argon2Params | None = None) -> E2eKey`: makes the header on first use; + raises `E2eLockedError` on a missing or wrong passphrase. Then + `Sealer(E2eCipher(key.master), layout)` and `Rtdb(..., root=layout.root, + sealer=sealer)`. +- `RequestServer(rtdb: Rtdb, worker: str, handlers: Mapping[str, Handler], + now: Callable[[], float] = time.time)` with `Handler = Callable[[dict[str, + Any]], Awaitable[dict[str, Any]]]` (request `body` in, result data out). + `await server.run()` watches `requests` until cancelled; `await + server.answer(rid, raw_node)` serves one. A handler refuses with + `raise RequestError(code, message, retry=False)`. Guards: unknown kind -> + `unknown_kind`; unsealed or undecryptable `req` -> `unsealed`; a handler + bug -> `internal` (logged); long handlers renew the lease every 60 s. +- `await presence.beat_forever(rtdb, worker, version)`: a beat every 30 s. +- `sse.listen(rtdb, path) -> AsyncGenerator[sse.Event]` (reconnects with + backoff, opens sealed units); `mirror.apply(tree, event) -> dict` keeps a + local copy of a streamed node. To publish a sealed snapshot node, `await + rtdb.put("", value)`. + +## Dart API (the Flutter client) + +Not extracted yet: the tutor app's `lib/sync/` and `lib/relay/` still hold +the client half. Until it moves here, a new client follows the wire format +above (`requests/{rid}/req` sealed with crdt_sync's Dart `E2eCipher`, then +listen on `requests/{rid}/result`). + +## Develop + +`uv venv --python 3.15 .venv && uv pip install -e . pytest==9.2.0 +pytest-cov==7.1.0 ruff==0.17.4 mypy==2.4.1`, then `scripts/run_subproject_tests.sh rtdb_relay` +from the repo root (100 % branch coverage), `ruff check .` and `mypy rtdb_relay`. diff --git a/rtdb_relay/pyproject.toml b/rtdb_relay/pyproject.toml new file mode 100644 index 0000000..396e66d --- /dev/null +++ b/rtdb_relay/pyproject.toml @@ -0,0 +1,176 @@ +[project] +name = "rtdb-relay" +version = "0.1.0" +description = "Sealed request relay over Firebase RTDB: PC worker (Python) + Flutter client (Dart)" +requires-python = ">=3.14" +dependencies = [ + "crdt-sync @ git+https://github.com/kuhyx/utils@crdt-sync-v0.11.1#subdirectory=crdt-sync", + "httpx==0.28.1", +] + +# Ship the library and its tests (consumers run the contract vectors). +[tool.setuptools] +packages = ["rtdb_relay", "rtdb_relay.tests"] + +[tool.ruff] +target-version = "py314" +include = ["*.py", "**/*.py"] +exclude = [".git", ".venv", "__pycache__", "build", "dist", ".eggs"] + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "D203", + "D213", + "COM812", + "ISC001", + "S603", +] +fixable = ["ALL"] +unfixable = [] + +[tool.ruff.lint.per-file-ignores] +"**/tests/*.py" = ["ANN", "ARG", "CPY001", "D", "EM101", "PLC0415", "PLR2004", "S101", "SLF001", "TC002"] +"**/_fake_*.py" = ["ANN401", "ARG002"] +"**/test_*.py" = ["ANN", "ARG", "CPY001", "D", "EM101", "PLC0415", "PLR2004", "S101", "SLF001", "TC002"] + + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +force-single-line = false +force-sort-within-sections = true +known-first-party = ["rtdb_relay"] + +[tool.ruff.lint.flake8-quotes] +docstring-quotes = "double" +inline-quotes = "double" + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +docstring-code-format = true + +[tool.mypy] +python_version = "3.14" +strict = true +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +# crdt_sync ships no py.typed: its names would all be "unfollowed". +disallow_any_unimported = false +disallow_any_explicit = false +disallow_any_generics = true +disallow_subclassing_any = true +strict_equality = true +extra_checks = true +ignore_missing_imports = true +show_error_codes = true +color_output = true +exclude = [".venv/"] + +[tool.pylint.main] +# The repo root on the path, so pylint resolves the package's own imports the +# same way pytest does instead of reporting import-error for them. +init-hook = "import sys; sys.path.insert(0, 'rtdb_relay')" +analyse-fallback-blocks = true +persistent = true +jobs = 0 +py-version = "3.14" +ignore = [".venv", "__pycache__"] +# Test suites intentionally use patterns (protected-access, fixture-arg +# shadowing) that don't apply to source code; ruff's per-file-ignores carry +# the equivalent exemptions, so tests stay linted -- by ruff. This is the same +# source-only pylint scope wake-alarm / diet-guard / testsAndMisc use. It has +# to be `ignore-paths`, not `ignore = ["tests"]`: `ignore` only matches the +# basename of what pylint is *given*, and pre-commit hands it explicit file +# paths, so a directory name in `ignore` never fires under the hook. +ignore-paths = ["^(.*/)?tests/.*$"] +ignore-patterns = [".*\\.pyi$"] + +[tool.pylint.basic] +# snake_case, plus the ``do_`` names http.server dispatches on: the +# stdlib looks handlers up as ``"do_" + command``, so ``do_GET`` is the only +# spelling that works and cannot be renamed to satisfy the style. +method-rgx = "([^\\W\\dA-Z][^\\WA-Z]{2,}|_[^\\WA-Z]*|__[^\\W\\dA-Z][^\\WA-Z]+__|do_[A-Z]+|of)$" + +[tool.pylint.messages_control] +enable = "all" +disable = [] + +[tool.pylint.design] +min-public-methods = 0 +max-module-lines = 1000 +max-attributes = 10 + +[tool.pylint.typecheck] +generated-members = [ + ".*\\.assert_called_once_with", + ".*\\.assert_called_once", + ".*\\.assert_called", + ".*\\.assert_not_called", + ".*\\.assert_any_call", + ".*\\.call_args", + ".*\\.call_args_list", + ".*\\.call_count", +] + +[tool.bandit] +exclude_dirs = ["tests", ".venv"] + +[tool.pytest.ini_options] +testpaths = ["rtdb_relay/tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", + "--strict-config", + "-ra", + "--cov=rtdb_relay", + "--cov-branch", + "--cov-report=term-missing", + "--cov-report=lcov", +] +filterwarnings = [ + "error", + "ignore::DeprecationWarning", + "default::pytest.PytestUnraisableExceptionWarning", +] + +[tool.coverage.run] +source = ["rtdb_relay"] +branch = true +omit = [ + "*/__pycache__/*", + "*/tests/*", + "*/.venv/*", +] + +[tool.coverage.report] +fail_under = 100 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "raise NotImplementedError", + "raise AssertionError", + "if TYPE_CHECKING:", + 'if __name__ == "__main__":', +] +partial_branches = ["pragma: no branch"] diff --git a/rtdb_relay/rtdb_relay/__init__.py b/rtdb_relay/rtdb_relay/__init__.py new file mode 100644 index 0000000..17463f8 --- /dev/null +++ b/rtdb_relay/rtdb_relay/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""A sealed request relay over Firebase RTDB (PC worker side). + +See ``README.md`` for the API and the wire format. +""" diff --git a/rtdb_relay/rtdb_relay/layout.py b/rtdb_relay/rtdb_relay/layout.py new file mode 100644 index 0000000..3538671 --- /dev/null +++ b/rtdb_relay/rtdb_relay/layout.py @@ -0,0 +1,81 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Which RTDB paths of an app's tree are clear, sealed units, or structure. + +The tree keeps its shape: keys stay clear so ``orderBy="$key"``, PATCH and +SSE paths work unchanged. Every *unit* (a node one writer writes whole, +below which nothing is written) is stored sealed; a unit may keep a few +clear siblings (server timestamps RTDB must resolve). ``clear`` prefixes +hold ids, counters and lease times only. Any other path is refused by the +sealer (fail closed), so a new node cannot go out in plaintext by accident. + +Patterns are ``/``-separated; ``*`` matches one segment. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +class E2eLayoutError(ValueError): + """A write the layout has no rule for, or one below a sealed unit.""" + + +def parts_of(path: str) -> list[str]: + """``path``'s segments (no empty ones).""" + return [p for p in path.split("/") if p] + + +def _matches(pattern: list[str], parts: list[str]) -> bool: + return len(pattern) == len(parts) and all( + p in {"*", q} for p, q in zip(pattern, parts, strict=True) + ) + + +@dataclass(frozen=True) +class Rule: + """Where ``parts`` falls: ``clear``, ``unit``, ``below``, ``struct``, ``unknown``. + + ``depth`` is the unit's length for ``below``; ``keep`` the clear fields + of a unit. + """ + + kind: str + depth: int = 0 + keep: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class Layout: + """An app's tree rules: ``root`` (e.g. ``tutor/v1-e2e``) plus patterns. + + ``clear`` patterns cover their whole subtree; ``units`` map a unit + pattern to the clear fields it keeps beside the sealed blob. + """ + + root: str + clear: tuple[str, ...] + units: dict[str, frozenset[str]] = field(default_factory=dict) + + def classify(self, parts: list[str]) -> Rule: + """The rule for the node at ``parts`` (relative to :attr:`root`).""" + for pattern in self.clear: + pat = pattern.split("/") + if len(pat) <= len(parts) and _matches(pat, parts[: len(pat)]): + return Rule("clear") + for pattern, keep in self.units.items(): + pat = pattern.split("/") + if _matches(pat, parts): + return Rule("unit", len(pat), keep) + if len(pat) < len(parts) and _matches(pat, parts[: len(pat)]): + return Rule("below", len(pat), keep) + for pattern in (*self.clear, *self.units): + pat = pattern.split("/") + if len(parts) < len(pat) and _matches(pat[: len(parts)], parts): + return Rule("struct") + return Rule("unknown") + + def unit_path(self, path: str) -> str | None: + """The sealed unit ``path`` lies strictly below, or ``None``.""" + parts = parts_of(path) + rule = self.classify(parts) + return "/".join(parts[: rule.depth]) if rule.kind == "below" else None diff --git a/rtdb_relay/rtdb_relay/lease.py b/rtdb_relay/rtdb_relay/lease.py new file mode 100644 index 0000000..00c639a --- /dev/null +++ b/rtdb_relay/rtdb_relay/lease.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Request ids and the claim lease on ``requests/{rid}/claim``. + +The claim node is ``{"by", "lease_until", "attempt", "phase", "at"}`` (ms), +the format the tutor's worker writes. One :class:`Lease` per request is the +only writer of its claim: it holds the current ETag, so renewals never race +each other into a 412 of our own making. +""" + +from __future__ import annotations + +import logging +import re +from typing import TYPE_CHECKING, Any, Final + +from rtdb_relay.rtdb import ConflictError + +if TYPE_CHECKING: + from collections.abc import Callable + + from rtdb_relay.rtdb import Rtdb + +RID: Final = re.compile(r"[0-9A-HJKMNP-TV-Z]{26}", re.ASCII) +LEASE_S: Final = 300.0 +RENEW_S: Final = 60.0 +_logger = logging.getLogger(__name__) + + +def valid_rid(rid: str) -> bool: + """Whether ``rid`` is a ULID (the only keys a path is built from).""" + return bool(RID.fullmatch(rid)) + + +def ms(t: float) -> int: + """Unix seconds -> unix ms.""" + return int(t * 1000) + + +def claimable(claim: object, now: float, own: str = "") -> bool: + """Null, a lease that ran out, or ``own`` (a dead process of this worker). + + The caller has checked there is no result. + """ + if not isinstance(claim, dict): + return True + lease = claim.get("lease_until") + mine = bool(own) and claim.get("by") == own + return mine or not isinstance(lease, (int, float)) or lease < ms(now) + + +class Lease: + """A claim this worker holds: attempt number, phase and current ETag.""" + + def __init__( + self, rtdb: Rtdb, rid: str, worker: str, now: Callable[[], float] + ) -> None: + """Not yet taken; see :meth:`take`.""" + self._rtdb, self.rid, self._worker, self._now = rtdb, rid, worker, now + self.attempt = 0 + self.phase = "claimed" + self._etag = "" + + @property + def path(self) -> str: + """``requests/{rid}/claim``.""" + return f"requests/{self.rid}/claim" + + def _node(self) -> dict[str, Any]: + now = self._now() + return { + "by": self._worker, + "lease_until": ms(now + LEASE_S), + "attempt": self.attempt, + "phase": self.phase, + "at": ms(now), + } + + async def take(self) -> bool: + """Claim the request (False: someone else holds it, or lost the race).""" + current, etag = await self._rtdb.get_etag(self.path) + if not claimable(current, self._now(), self._worker): + return False + before = current.get("attempt", 0) if isinstance(current, dict) else 0 + self.attempt = (before if isinstance(before, int) else 0) + 1 + try: + _, self._etag = await self._rtdb.put_etag(self.path, self._node(), etag) + except ConflictError: + _logger.warning("another worker claimed %s first", self.rid) + return False + return True + + async def renew(self, phase: str | None = None) -> None: + """Extend the lease (and move to ``phase``); a lost race is only logged.""" + if phase is not None: + self.phase = phase + try: + _, self._etag = await self._rtdb.put_etag( + self.path, self._node(), self._etag + ) + except ConflictError: + _logger.warning("claim on %s lost to another writer", self.rid) diff --git a/rtdb_relay/rtdb_relay/mirror.py b/rtdb_relay/rtdb_relay/mirror.py new file mode 100644 index 0000000..c51296a --- /dev/null +++ b/rtdb_relay/rtdb_relay/mirror.py @@ -0,0 +1,46 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""A local copy of a streamed RTDB node, kept current from put/patch events.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from rtdb_relay.sse import Event + + +def _parts(path: str) -> list[str]: + return [p for p in path.split("/") if p] + + +def _set(tree: dict[str, Any], parts: list[str], value: object) -> dict[str, Any]: + """``tree`` with ``value`` at ``parts`` (``None`` deletes, empty parents go).""" + head, rest = parts[0], parts[1:] + if rest: + child = tree.get(head) + value = _set(child if isinstance(child, dict) else {}, rest, value) + if not value: + value = None + if value is None: + tree.pop(head, None) + else: + tree[head] = value + return tree + + +def apply(tree: dict[str, Any], event: Event) -> dict[str, Any]: + """The mirrored node after ``event`` (a new dict on a root ``put``).""" + parts = _parts(event.path) + if event.kind == "put": + if not parts: + return event.data if isinstance(event.data, dict) else {} + return _set(tree, parts, event.data) + for key, value in (event.data or {}).items(): + tree = _set(tree, [*parts, *_parts(key)], value) + return tree + + +def first_key(path: str) -> str | None: + """The top-level key an event path touches (``None`` for the root).""" + parts = _parts(path) + return parts[0] if parts else None diff --git a/rtdb_relay/rtdb_relay/presence.py b/rtdb_relay/rtdb_relay/presence.py new file mode 100644 index 0000000..76d2214 --- /dev/null +++ b/rtdb_relay/rtdb_relay/presence.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""The worker's heartbeat at ``presence/workers/{worker}``. + +Clients judge the worker alive from when beats stop arriving (``beat`` is an +RTDB server timestamp, kept clear beside the sealed unit), never by +comparing it to their own clock. +""" + +from __future__ import annotations + +import asyncio +import socket +from typing import TYPE_CHECKING, Final + +from rtdb_relay.rtdb import SERVER_TIME + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + from rtdb_relay.rtdb import Rtdb + +BEAT_S: Final = 30.0 + + +def node(version: str) -> dict[str, object]: + """One beat: server time, the host and the worker's build.""" + return {"beat": SERVER_TIME, "host": socket.gethostname(), "version": version} + + +async def beat_forever( + rtdb: Rtdb, + worker: str, + version: str, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, +) -> None: + """Write a beat every :data:`BEAT_S` until cancelled.""" + while True: + await rtdb.put(f"presence/workers/{worker}", node(version)) + await sleep(BEAT_S) diff --git a/rtdb_relay/rtdb_relay/rtdb.py b/rtdb_relay/rtdb_relay/rtdb.py new file mode 100644 index 0000000..819fe15 --- /dev/null +++ b/rtdb_relay/rtdb_relay/rtdb.py @@ -0,0 +1,197 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""A minimal async RTDB REST client: GET/PUT/PATCH/DELETE, ETags, server time. + +Under the app's root every value goes through a ``seal.Sealer`` (sealed on +write, opened on read) once ``sealer`` is set; ``at()`` gives the raw +client the e2e header and one-time copies use. + +Every call authenticates with ``?auth=`` (RTDB REST has no header +form), so nothing here ever formats a URL or an ``httpx`` error into a +message or a log line: errors name the method, the path and the status only. +httpx's own request log (INFO, full URL) is silenced for the same reason. +""" + +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, Any, Final +from urllib.parse import quote + +import httpx + +from rtdb_relay.layout import parts_of + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable + + from rtdb_relay.seal import Sealer + +SERVER_TIME: Final = {".sv": "timestamp"} +NULL_ETAG: Final = "null_etag" +_PRECONDITION_FAILED: Final = 412 +_TIMEOUT: Final = httpx.Timeout(30.0, connect=15.0) +# The SSE read timeout: RTDB sends keep-alive every 30 s. +_STREAM_TIMEOUT: Final = httpx.Timeout(30.0, connect=15.0, read=75.0) + +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) + + +class RtdbError(RuntimeError): + """A request failed; the message never contains the URL or the token.""" + + +class ConflictError(RtdbError): + """A conditional write lost the race (HTTP 412): the current value and ETag.""" + + def __init__(self, path: str, value: object, etag: str) -> None: + """Keep what the server holds now.""" + super().__init__(f"conditional write lost on {path}") + self.value = value + self.etag = etag + + +class Rtdb: + """REST calls under ``root``; ``token`` yields a fresh id token.""" + + def __init__( + self, + database_url: str, + token: Callable[[], Awaitable[str]], + client: httpx.AsyncClient | None = None, + *, + root: str, + sealer: Sealer | None = None, + ) -> None: + """Bind the database; ``client`` is injectable (tests use a fake).""" + self._base = database_url.rstrip("/") + self._token = token + self.client = client or httpx.AsyncClient(timeout=_TIMEOUT) + self.root = root + self.sealer = sealer + + def at(self, root: str) -> Rtdb: + """A raw (unsealed) client on ``root`` sharing this one's connection.""" + return type(self)(self._base, self._token, self.client, root=root) + + def url(self, path: str) -> str: + """The REST URL of ``path`` (relative to the root), no token.""" + parts = [*parts_of(self.root), *parts_of(path)] + return f"{self._base}/{'/'.join(quote(p, safe='') for p in parts)}.json" + + def _seal(self, path: str, value: object) -> object: + return value if self.sealer is None else self.sealer.seal(path, value) + + def opened(self, path: str, value: object) -> object: + """``value`` read at ``path`` with its sealed units opened.""" + return value if self.sealer is None else self.sealer.open(path, value) + + async def params(self, extra: dict[str, str] | None = None) -> dict[str, str]: + """Query parameters: the auth token plus ``extra``.""" + return {"auth": await self._token(), **(extra or {})} + + async def _call( + self, + method: str, + path: str, + *, + value: object = None, + headers: dict[str, str] | None = None, + params: dict[str, str] | None = None, + ) -> httpx.Response: + kwargs: dict[str, Any] = {"params": await self.params(params)} + if method in {"PUT", "PATCH"}: + kwargs["content"] = json.dumps(value).encode() # json=None sends no body + try: + response = await self.client.request( + method, self.url(path), headers=headers or {}, **kwargs + ) + except httpx.HTTPError as err: + msg = f"{method} {path}: {type(err).__name__}" + raise RtdbError(msg) from None + if response.status_code == _PRECONDITION_FAILED: + current = self.opened(path, response.json()) + raise ConflictError(path, current, response.headers.get("etag", "")) + if response.is_error: + msg = f"{method} {path}: HTTP {response.status_code}" + raise RtdbError(msg) + return response + + async def get(self, path: str, params: dict[str, str] | None = None) -> object: + """The value at ``path`` (``None`` when absent). + + Below a sealed unit the whole unit is read and opened, then walked. + """ + unit = None if self.sealer is None else self.sealer.unit_path(path) + if unit is not None: + node: Any = await self.get(unit) + for part in parts_of(path)[len(parts_of(unit)) :]: + node = node.get(part) if isinstance(node, dict) else None + return node + value: object = (await self._call("GET", path, params=params)).json() + return self.opened(path, value) + + async def get_etag(self, path: str) -> tuple[Any, str]: + """The value at ``path`` and its ETag (``null_etag`` when absent).""" + response = await self._call("GET", path, headers={"X-Firebase-ETag": "true"}) + value = self.opened(path, response.json()) + return value, response.headers.get("etag", NULL_ETAG) + + async def put(self, path: str, value: object) -> object: + """Write ``value``; returns it as stored (server timestamps resolved).""" + sealed = self._seal(path, value) + stored: object = (await self._call("PUT", path, value=sealed)).json() + return self.opened(path, stored) + + async def put_etag( + self, path: str, value: object, if_match: str + ) -> tuple[Any, str]: + """A conditional PUT that also returns the new ETag.""" + headers = {"if-match": if_match, "X-Firebase-ETag": "true"} + sealed = self._seal(path, value) + response = await self._call("PUT", path, value=sealed, headers=headers) + return self.opened(path, response.json()), response.headers.get("etag", "") + + async def patch(self, path: str, value: dict[str, Any]) -> None: + """Merge ``value``'s children into ``path`` (``None`` children delete).""" + if self.sealer is not None: + value = self.sealer.seal_patch(path, value) + await self._call("PATCH", path, value=value) + + async def delete(self, path: str, if_match: str | None = None) -> None: + """Remove ``path``; with ``if_match`` only if unchanged.""" + headers = {"if-match": if_match} if if_match else {} + await self._call("DELETE", path, headers=headers) + + async def open_stream( + self, path: str, params: dict[str, str] | None = None + ) -> AsyncIterator[str]: + """The raw SSE lines of one connection to ``path`` (see ``sse``).""" + request = self.client.build_request( + "GET", + self.url(path), + params=await self.params(params), + headers={"Accept": "text/event-stream"}, + timeout=_STREAM_TIMEOUT, + ) + try: + response = await self.client.send(request, stream=True) + except httpx.HTTPError as err: + msg = f"STREAM {path}: {type(err).__name__}" + raise RtdbError(msg) from None + try: + if response.is_error: + msg = f"STREAM {path}: HTTP {response.status_code}" + raise RtdbError(msg) + async for line in response.aiter_lines(): + yield line + except httpx.HTTPError as err: + msg = f"STREAM {path}: {type(err).__name__}" + raise RtdbError(msg) from None + finally: + await response.aclose() + + async def aclose(self) -> None: + """Close the HTTP client.""" + await self.client.aclose() diff --git a/rtdb_relay/rtdb_relay/seal.py b/rtdb_relay/rtdb_relay/seal.py new file mode 100644 index 0000000..9788980 --- /dev/null +++ b/rtdb_relay/rtdb_relay/seal.py @@ -0,0 +1,133 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Sealing writes and opening reads of an app's RTDB tree. + +A unit is stored as ``{"_e2e": , }``; +the blob is crdt_sync's ``E2eCipher`` with the unit's full path as AAD, so +a blob copied to another path does not open. The byte format is pinned by +``tests/data/e2e_vectors.json`` (the tutor's contract vectors). +""" + +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, Any, Final + +from crdt_sync import E2eCipher, E2eDecryptError + +from rtdb_relay.layout import E2eLayoutError, Layout, parts_of + +if TYPE_CHECKING: + from rtdb_relay.layout import Rule + +SEALED: Final = "_e2e" +_logger = logging.getLogger(__name__) + + +def pruned(value: object) -> object: + """``value`` as RTDB would keep it: no null members, no empty objects.""" + if not isinstance(value, dict): + return value + out = {k: pruned(v) for k, v in value.items()} + return {k: v for k, v in out.items() if v is not None} or None + + +def dumps(value: object) -> str: + """Compact JSON, UTF-8 kept (the sealed plaintext format).""" + return json.dumps(value, separators=(",", ":"), ensure_ascii=False) + + +class Sealer: + """Seals writes and opens reads of the tree ``layout`` describes.""" + + def __init__( + self, cipher: E2eCipher, layout: Layout, root: str | None = None + ) -> None: + """Use ``cipher`` for every unit; AAD prefix ``root`` (default the layout's).""" + self._cipher = cipher + self.layout = layout + self.root = layout.root if root is None else root + + def classify(self, parts: list[str]) -> Rule: + """The layout's rule for ``parts``.""" + return self.layout.classify(parts) + + def unit_path(self, path: str) -> str | None: + """The sealed unit ``path`` lies strictly below, or ``None``.""" + return self.layout.unit_path(path) + + def _aad(self, parts: list[str]) -> str: + return "/".join([self.root, *parts]) + + # -- writes --------------------------------------------------------------- + def seal(self, path: str, value: object) -> object: + """``value`` as it may be stored at ``path``; refuses unknown paths.""" + return self._seal(parts_of(path), value) + + def seal_patch(self, path: str, body: dict[str, Any]) -> dict[str, Any]: + """A PATCH body for ``path``: each (multi-segment) key sealed on its own.""" + if self.classify(parts_of(path)).kind in {"unit", "below"}: + msg = f"PATCH into sealed unit {path}" + raise E2eLayoutError(msg) + base = parts_of(path) + return {k: self._seal([*base, *parts_of(k)], v) for k, v in body.items()} + + def _seal(self, parts: list[str], value: object) -> object: + rule = self.classify(parts) + if rule.kind == "clear" or value is None: + return value + if rule.kind == "unit": + return self._seal_unit(parts, value, rule.keep) + if rule.kind == "struct" and isinstance(value, dict): + return {k: self._seal([*parts, *parts_of(k)], v) for k, v in value.items()} + where = "/".join(parts) or "/" + msg = f"no e2e rule for a {type(value).__name__} at {where} ({rule.kind})" + raise E2eLayoutError(msg) + + def _seal_unit( + self, parts: list[str], value: object, keep: frozenset[str] + ) -> object: + value = pruned(value) + if value is None: + return None # RTDB stores no null and no empty object + clear: dict[str, Any] = {} + if isinstance(value, dict) and keep: + clear = {k: v for k, v in value.items() if k in keep} + value = {k: v for k, v in value.items() if k not in keep} + blob = self._cipher.seal(dumps(value), path=self._aad(parts)) + return {SEALED: blob, **clear} + + # -- reads ---------------------------------------------------------------- + def open(self, path: str, value: object) -> object: + """``value`` read at ``path``, every unit in it opened. + + A unit that does not open (tampered, other key) reads as absent. + Legacy plaintext (a unit without ``_e2e``) is returned as stored. + """ + return self._open(parts_of(path), value) + + def _open(self, parts: list[str], value: object) -> object: + rule = self.classify(parts) + if rule.kind == "unit": + return self._open_unit(parts, value) + if rule.kind == "below": + msg = f"read below sealed unit: {'/'.join(parts)}" + raise E2eLayoutError(msg) + if rule.kind == "struct" and isinstance(value, dict): + out = {k: self._open([*parts, k], v) for k, v in value.items()} + return {k: v for k, v in out.items() if v is not None} + if rule.kind == "struct" and isinstance(value, list): + return [self._open([*parts, str(i)], v) for i, v in enumerate(value)] + return value + + def _open_unit(self, parts: list[str], value: object) -> object: + if not isinstance(value, dict) or not isinstance(value.get(SEALED), str): + return value # legacy plaintext, accepted during a migration + clear = {k: v for k, v in value.items() if k != SEALED} + try: + text = self._cipher.open(value[SEALED], path=self._aad(parts)) + data = json.loads(text) + except E2eDecryptError, ValueError: + _logger.warning("undecryptable node %s dropped", "/".join(parts)) + return None + return {**data, **clear} if isinstance(data, dict) else data diff --git a/rtdb_relay/rtdb_relay/server.py b/rtdb_relay/rtdb_relay/server.py new file mode 100644 index 0000000..8f62b0c --- /dev/null +++ b/rtdb_relay/rtdb_relay/server.py @@ -0,0 +1,151 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Serve ``requests/{rid}`` with registered kind handlers (the PC side). + +A client writes ``requests/{rid}/req`` = ``{"kind", "dev", "created", +"sent_at", "body"}`` (sealed). The server claims it (:class:`Lease`), runs +the handler for ``kind`` and writes ``requests/{rid}/result`` = +``{"ok": true, "kind", "at", **data}`` or ``{"ok": false, "kind", "code", +"error", "retry", "at"}``. Guards (fail closed): + +- only kinds in ``handlers`` run; any other kind gets ``unknown_kind``; +- an unsealed (plaintext) or undecryptable ``req`` gets ``unsealed``; +- a non-ULID rid is never used in a path (skipped); +- a handler bug becomes a visible ``internal`` result, never a silent drop. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +from typing import TYPE_CHECKING, Any, Final + +from rtdb_relay import mirror, sse +from rtdb_relay.lease import RENEW_S, Lease, valid_rid +from rtdb_relay.rtdb import SERVER_TIME +from rtdb_relay.seal import SEALED + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Mapping + + from rtdb_relay.rtdb import Rtdb + +type Handler = Callable[[dict[str, Any]], Awaitable[dict[str, Any]]] +_logger = logging.getLogger(__name__) +UNKNOWN_KIND: Final = "unknown_kind" +UNSEALED: Final = "unsealed" +INTERNAL: Final = "internal" + + +class RequestError(Exception): + """A handler's refusal: ``code`` and ``message`` reach the client.""" + + def __init__(self, code: str, message: str, *, retry: bool = False) -> None: + """Fail the request with ``code``; ``retry`` tells the client it may.""" + super().__init__(message) + self.code, self.message, self.retry = code, message, retry + + +def failure( + kind: object, code: str, message: str, *, retry: bool = False +) -> dict[str, Any]: + """A failed ``result`` node (``at`` resolved by the server).""" + return { + "ok": False, + "kind": kind if isinstance(kind, str) else "", + "code": code, + "error": message, + "retry": retry, + "at": SERVER_TIME, + } + + +class RequestServer: + """Claims and answers requests under ``rtdb``'s root.""" + + def __init__( + self, + rtdb: Rtdb, + worker: str, + handlers: Mapping[str, Handler], + now: Callable[[], float] = time.time, + ) -> None: + """Serve with ``rtdb`` (sealed client); ``worker`` names this process.""" + self.rtdb, self.worker, self.handlers, self._now = rtdb, worker, handlers, now + self._raw = rtdb.at(rtdb.root) + self._busy: set[str] = set() + self.tasks: set[asyncio.Task[None]] = set() + + def request_of(self, rid: str, raw: object) -> dict[str, Any] | None: + """The opened ``req``, or ``None`` when it was not sealed or not ours.""" + if not isinstance(raw, dict) or not isinstance(raw.get(SEALED), str): + return None + req = self.rtdb.opened(f"requests/{rid}/req", raw) + return req if isinstance(req, dict) and SEALED not in req else None + + async def answer(self, rid: str, node: dict[str, Any]) -> dict[str, Any] | None: + """Serve one pending request node; ``None`` when not ours to answer.""" + if not valid_rid(rid) or "result" in node or "req" not in node: + return None + req = self.request_of(rid, node["req"]) + kind = req.get("kind") if req else None + lease = Lease(self.rtdb, rid, self.worker, self._now) + if not await lease.take(): + return None + if req is None: + result = failure(kind, UNSEALED, "request was not sealed") + elif not isinstance(kind, str) or kind not in self.handlers: + result = failure(kind, UNKNOWN_KIND, f"no handler for {kind!r}") + else: + result = await self._run(lease, kind, req) + await self.rtdb.put(f"requests/{rid}/result", result) + return result + + async def _run( + self, lease: Lease, kind: str, req: dict[str, Any] + ) -> dict[str, Any]: + body = req.get("body") + keeper = asyncio.create_task(self._keep(lease)) + try: + data = await self.handlers[kind](body if isinstance(body, dict) else {}) + except RequestError as err: + return failure(kind, err.code, err.message, retry=err.retry) + except Exception as err: + _logger.exception("handler %s failed", kind) + return failure(kind, INTERNAL, type(err).__name__, retry=True) + finally: + keeper.cancel() + with contextlib.suppress(asyncio.CancelledError): + await keeper + return {**data, "ok": True, "kind": kind, "at": SERVER_TIME} + + async def _keep(self, lease: Lease) -> None: + while True: + await asyncio.sleep(RENEW_S) + await lease.renew("running") + + def dispatch(self, tree: dict[str, Any]) -> None: + """Start a task for every pending request not already in flight.""" + for rid, node in tree.items(): + if rid in self._busy or not isinstance(node, dict) or "result" in node: + continue + self._busy.add(rid) + task = asyncio.create_task(self._serve(rid, node)) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + async def _serve(self, rid: str, node: dict[str, Any]) -> None: + try: + await self.answer(rid, node) + except Exception: + _logger.exception("request %s failed", rid) + finally: + self._busy.discard(rid) + + async def run(self) -> None: + """Watch ``requests`` (raw, so sealing is checked) until cancelled.""" + tree: dict[str, Any] = {} + async for event in sse.listen(self._raw, "requests"): + tree = mirror.apply(tree, event) + self.dispatch(tree) diff --git a/rtdb_relay/rtdb_relay/sse.py b/rtdb_relay/rtdb_relay/sse.py new file mode 100644 index 0000000..7f1cc2e --- /dev/null +++ b/rtdb_relay/rtdb_relay/sse.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""RTDB streaming: Server-Sent Events, reconnected forever. + +Each (re)connect starts with a ``put`` of the whole node at path ``/``, so a +consumer that mirrors ``put``/``patch`` events (``mirror.apply``) is always +correct after a drop. ``auth_revoked`` (the id token expired) reconnects at +once with a fresh token; network errors back off; ``cancel`` (the rules now +refuse the path) is fatal. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +import json +import logging +from typing import TYPE_CHECKING, Any, Final + +from rtdb_relay.rtdb import RtdbError + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable + + from rtdb_relay.rtdb import Rtdb + +BACKOFF_S: Final = (1.0, 2.0, 5.0, 10.0, 30.0) +_logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Event: + """One ``put``/``patch`` event: where it applies and the new data.""" + + kind: str + path: str + data: Any + + +class StreamCancelledError(RtdbError): + """RTDB cancelled the stream (rules refuse the path): not retried.""" + + +def parse(lines: list[str]) -> tuple[str, str]: + """``(event, data)`` of one SSE block (its lines, blank line excluded).""" + event, data = "message", [] + for line in lines: + field, _, value = line.partition(":") + value = value.removeprefix(" ") + if field == "event": + event = value + elif field == "data": + data.append(value) + return event, "\n".join(data) + + +async def _blocks(source: AsyncIterator[str]) -> AsyncIterator[tuple[str, str]]: + block: list[str] = [] + async for line in source: + if line: + block.append(line) + elif block: + yield parse(block) + block = [] + + +def opened(rtdb: Rtdb, path: str, event: Event) -> Event: + """``event`` with its sealed units opened (``path``: the listened node). + + A ``put``/``patch`` never lands below a unit: units are written whole. + """ + where = f"{path.strip('/')}/{event.path.strip('/')}".strip("/") + if event.kind == "put": + return Event(event.kind, event.path, rtdb.opened(where, event.data)) + data = event.data if isinstance(event.data, dict) else {} + body = {k: rtdb.opened(f"{where}/{k}", v) for k, v in data.items()} + return Event(event.kind, event.path, body) + + +async def _one_connection( + rtdb: Rtdb, path: str, params: dict[str, str] | None +) -> AsyncIterator[Event]: + """Events of a single connection; returns on ``auth_revoked`` or EOF.""" + async for kind, raw in _blocks(rtdb.open_stream(path, params)): + if kind in {"put", "patch"}: + payload = json.loads(raw) + event = Event(kind, str(payload.get("path", "/")), payload.get("data")) + yield opened(rtdb, path, event) + elif kind == "auth_revoked": + _logger.info("stream %s: token expired, reconnecting", path) + return + elif kind == "cancel": + msg = f"stream {path} cancelled by the server" + raise StreamCancelledError(msg) + + +async def listen( + rtdb: Rtdb, + path: str, + params: dict[str, str] | None = None, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, +) -> AsyncGenerator[Event]: + """Every event at ``path`` across reconnects; runs until cancelled.""" + failures = 0 + while True: + try: + async for event in _one_connection(rtdb, path, params): + failures = 0 + yield event + except StreamCancelledError: + raise + except (RtdbError, ValueError) as err: # ValueError: a bad JSON payload + delay = BACKOFF_S[min(failures, len(BACKOFF_S) - 1)] + failures += 1 + _logger.warning("stream %s dropped (%s); retry in %.0fs", path, err, delay) + await sleep(delay) diff --git a/rtdb_relay/rtdb_relay/tests/__init__.py b/rtdb_relay/rtdb_relay/tests/__init__.py new file mode 100644 index 0000000..cb081c0 --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Tests of rtdb_relay; the fake RTDB and the tutor layout are reusable.""" diff --git a/rtdb_relay/rtdb_relay/tests/_fake_rtdb.py b/rtdb_relay/rtdb_relay/tests/_fake_rtdb.py new file mode 100644 index 0000000..bff710e --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/_fake_rtdb.py @@ -0,0 +1,235 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""An in-memory Firebase RTDB behind ``httpx.MockTransport``. + +REST semantics the relay relies on: GET (shallow, orderBy/startAt), PUT, +PATCH, DELETE, ``X-Firebase-ETag`` / ``if-match`` with 412 + current value, +``{".sv": "timestamp"}``, ``?auth=`` checking, and SSE streams (initial put, +put/patch events, keep-alive, auth_revoked, cancel, drop). + +The stored tree (``raw``) is what RTDB would hold: under the tutor layout +every unit is sealed with :data:`TEST_KEY`, as the relay writes it. +``get``/``set`` are the test's clear view (``set`` writes like a client +would: sealed); ``sealed=False`` stores and serves plaintext. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from typing import TYPE_CHECKING, Any + +from crdt_sync import E2eCipher +import httpx + +from rtdb_relay.layout import E2eLayoutError +from rtdb_relay.rtdb import Rtdb +from rtdb_relay.seal import Sealer +from rtdb_relay.tests._fake_tree import ( + _assign, + _event, + _parts, + _prune, + _reply, + etag, +) +from rtdb_relay.tests._layouts import TUTOR + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + +AUTH = "test-token" +DB = "https://db.test" +TEST_KEY = bytes(range(32)) +ROOT = TUTOR.root + + +def sealer() -> Sealer: + """The sealer of :data:`TEST_KEY`.""" + return Sealer(E2eCipher(TEST_KEY), TUTOR) + + +class FakeRtdb: + """The database; ``now_ms`` resolves server timestamps.""" + + def __init__(self, now_ms: int = 1_800_000_000_000, *, sealed: bool = True) -> None: + self.raw: dict[str, Any] = {} + self.sealer = sealer() if sealed else None + self.now_ms = now_ms + self.streams: list[tuple[list[str], asyncio.Queue[str | None]]] = [] + self.fail: dict[tuple[str, str], int] = {} # (method, rel path) -> status + self.raise_on: set[tuple[str, str]] = set() # -> httpx.ConnectError + self.log: list[tuple[str, str]] = [] + self.token = AUTH + + # -- the test's clear view ------------------------------------------------- + def _rel(self, path: str) -> list[str] | None: + parts = _parts(path) + root = _parts(ROOT) + return parts[len(root) :] if parts[: len(root)] == root else None + + def get(self, path: str) -> Any: + """The value at ``path``, sealed units opened.""" + rel = self._rel(path) + if self.sealer is None or rel is None: + return self.raw_get(path) + opened = self.sealer.open("", self.raw_get(ROOT)) + node: Any = opened + for part in rel: + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + def set(self, path: str, value: Any) -> None: + """Write ``value`` as a client would (sealed; inside a unit: resealed).""" + rel = self._rel(path) + if self.sealer is None or rel is None: + self.raw_set(path, value) + return + unit = TUTOR.unit_path("/".join(rel)) + if unit is None: + try: + sealed = self.sealer.seal("/".join(rel), self._sv(value)) + except E2eLayoutError as err: + # A mis-shaped node no sealer writes: stored as a hostile client would. + logging.getLogger(__name__).warning("stored unsealed: %s", err) + sealed = value + self.raw_set(path, sealed) + return + node = self.get(f"{ROOT}/{unit}") + node = node if isinstance(node, dict) else {} + _assign(node, rel[len(_parts(unit)) :], value) + self.raw_set(f"{ROOT}/{unit}", self.sealer.seal(unit, self._sv(node))) + + # -- tree ----------------------------------------------------------------- + def raw_get(self, path: str) -> Any: + """The stored value at ``path`` (ciphertext under the root).""" + node: Any = self.raw + for part in _parts(path): + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + def _sv(self, value: Any) -> Any: + if value == {".sv": "timestamp"}: + return self.now_ms + if isinstance(value, dict): + out = {k: self._sv(v) for k, v in value.items()} + return {k: v for k, v in out.items() if v is not None} or None + return value + + def raw_set(self, path: str, value: Any) -> None: + """Store ``value`` at ``path`` as given (server timestamps resolved).""" + parts = _parts(path) + value = self._sv(value) + if not parts: + self.raw = value if isinstance(value, dict) else {} + return + node = self.raw + for part in parts[:-1]: + if not isinstance(node.get(part), dict): + node[part] = {} + node = node[part] + if value is None: + node.pop(parts[-1], None) + else: + node[parts[-1]] = value + self.raw = _prune(self.raw) or {} + + # -- streams -------------------------------------------------------------- + def _notify(self, kind: str, path: str, data: Any) -> None: + where = _parts(path) + for listen, queue in self.streams: + if where[: len(listen)] == listen: + rel = "/" + "/".join(where[len(listen) :]) + queue.put_nowait(_event(kind, rel, data)) + elif listen[: len(where)] == where: + queue.put_nowait(_event("put", "/", self.raw_get("/".join(listen)))) + + def push(self, raw: str | None) -> None: + """Send a raw SSE block (or ``None``: close) to every stream.""" + for _, queue in self.streams: + queue.put_nowait(raw) + + # -- HTTP ----------------------------------------------------------------- + def client(self) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.MockTransport(self.handle)) + + def rtdb(self) -> Rtdb: + async def token() -> str: + return self.token + + return Rtdb(DB, token, client=self.client(), root=ROOT, sealer=self.sealer) + + async def handle(self, request: httpx.Request) -> httpx.Response: + full = request.url.path.removesuffix(".json").strip("/") + path = ( + "/" + full.removeprefix(ROOT).strip("/") + if full.startswith(ROOT) + else "/" + full + ) + method = request.method + self.log.append((method, path)) + if (method, path) in self.raise_on: + msg = "boom" + raise httpx.ConnectError(msg, request=request) + if request.url.params.get("auth") != AUTH: + return _reply(401, {"error": "Permission denied"}) + if (method, path) in self.fail: + return _reply(self.fail[(method, path)], {"error": "x"}) + if request.headers.get("accept") == "text/event-stream": + return self._stream(full) + current = self.raw_get(full) + match = request.headers.get("if-match") + if match is not None and match != etag(current): + return _reply(412, current, headers={"ETag": etag(current)}) + if method == "GET": + return self._get(request, full, current) + return self._write(method, full, request) + + def _write(self, method: str, full: str, request: httpx.Request) -> httpx.Response: + if method == "PUT": + self.raw_set(full, json.loads(request.content)) + self._notify("put", full, self.raw_get(full)) + stored = self.raw_get(full) + return _reply(200, stored, headers={"ETag": etag(stored)}) + if method == "PATCH": + body = json.loads(request.content) + for key, value in body.items(): + self.raw_set(f"{full}/{key}", value) + self._notify("patch", full, self._sv(body)) + return _reply(200, body) + self.raw_set(full, None) # DELETE + self._notify("put", full, None) + return _reply(200, None) + + def _get(self, request: httpx.Request, full: str, value: Any) -> httpx.Response: + params = request.url.params + if params.get("shallow") == "true" and isinstance(value, dict): + value = dict.fromkeys(value, True) + start = params.get("startAt") + if start and isinstance(value, dict): + value = {k: v for k, v in value.items() if k >= json.loads(start)} + headers = ( + {"ETag": etag(value)} if request.headers.get("x-firebase-etag") else {} + ) + return _reply(200, value, headers=headers) + + def _stream(self, full: str) -> httpx.Response: + queue: asyncio.Queue[str | None] = asyncio.Queue() + entry = (_parts(full), queue) + self.streams.append(entry) + queue.put_nowait(_event("put", "/", self.raw_get(full))) + + async def body() -> AsyncIterator[bytes]: + try: + while (block := await queue.get()) is not None: + yield block.encode() + finally: + self.streams.remove(entry) + + return httpx.Response( + 200, content=body(), headers={"content-type": "text/event-stream"} + ) diff --git a/rtdb_relay/rtdb_relay/tests/_fake_tree.py b/rtdb_relay/rtdb_relay/tests/_fake_tree.py new file mode 100644 index 0000000..954c0fc --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/_fake_tree.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Tree and wire helpers of the fake RTDB (``_fake_rtdb``).""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +import httpx + + +def _parts(path: str) -> list[str]: + return [p for p in path.strip("/").split("/") if p] + + +def etag(value: object) -> str: + if value is None: + return "null_etag" + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def _event(kind: str, path: str, data: Any) -> str: + return f"event: {kind}\ndata: {json.dumps({'path': path, 'data': data})}\n\n" + + +def _assign(node: dict[str, Any], parts: list[str], value: Any) -> None: + for part in parts[:-1]: + if not isinstance(node.get(part), dict): + node[part] = {} + node = node[part] + node[parts[-1]] = value + + +def _prune(node: Any) -> Any: + """RTDB keeps no empty objects.""" + if not isinstance(node, dict): + return node + out = {k: _prune(v) for k, v in node.items()} + return {k: v for k, v in out.items() if v is not None and v != {}} or None + + +def _reply( + status: int, value: Any, headers: dict[str, str] | None = None +) -> httpx.Response: + """A JSON response; ``httpx.Response(json=None)`` would send no body at all.""" + return httpx.Response( + status, content=json.dumps(value).encode(), headers=headers or {} + ) diff --git a/rtdb_relay/rtdb_relay/tests/_layouts.py b/rtdb_relay/rtdb_relay/tests/_layouts.py new file mode 100644 index 0000000..6591819 --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/_layouts.py @@ -0,0 +1,40 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""The tutor's layout (its DOCS-contract §12): the reference the vectors pin.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from rtdb_relay.layout import Layout + +VECTORS: dict[str, Any] = json.loads( + (Path(__file__).parent / "data" / "e2e_vectors.json").read_text() +) +TUTOR = Layout( + root="tutor/v1-e2e", + clear=("meta", "rev", "migration", "requests/*/claim", "sessions/*/progress/rid"), + units={ + "live": frozenset(), + "prefs/*": frozenset(), + "stats/*": frozenset(), + "presence/workers/*": frozenset({"beat"}), + "presence/clients/*": frozenset({"seen"}), + "uploads/*/*": frozenset(), + "requests/*/req": frozenset({"created"}), + "requests/*/result": frozenset({"at"}), + "sessions/*/meta/*": frozenset(), + "sessions/*/msgs/*": frozenset(), + "sessions/*/receipts/*": frozenset(), + "sessions/*/user/*": frozenset(), + "sessions/*/progress/steps/*": frozenset(), + "session_index/*": frozenset(), + "subjects/*/meta": frozenset(), + "subjects/*/plan": frozenset(), + "subjects/*/cards": frozenset(), + "subjects/*/concepts": frozenset(), + "log/*/*": frozenset(), + "feedback/*": frozenset(), + }, +) diff --git a/rtdb_relay/rtdb_relay/tests/data/e2e_vectors.json b/rtdb_relay/rtdb_relay/tests/data/e2e_vectors.json new file mode 100644 index 0000000..5ab7a69 --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/data/e2e_vectors.json @@ -0,0 +1,266 @@ +{ + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "root": "tutor/v1-e2e", + "sealed_key": "_e2e", + "rules": [ + { + "path": "", + "kind": "struct" + }, + { + "path": "meta", + "kind": "clear" + }, + { + "path": "meta/min_client", + "kind": "clear" + }, + { + "path": "rev", + "kind": "clear" + }, + { + "path": "rev/catalog", + "kind": "clear" + }, + { + "path": "live", + "kind": "unit" + }, + { + "path": "prefs", + "kind": "struct" + }, + { + "path": "prefs/current_subject", + "kind": "unit" + }, + { + "path": "stats", + "kind": "struct" + }, + { + "path": "stats/today", + "kind": "unit" + }, + { + "path": "stats/today/new_cap", + "kind": "below" + }, + { + "path": "presence/workers/d1", + "kind": "unit" + }, + { + "path": "presence/clients/d1", + "kind": "unit" + }, + { + "path": "uploads/u1/c0001", + "kind": "unit" + }, + { + "path": "requests", + "kind": "struct" + }, + { + "path": "requests/r1", + "kind": "struct" + }, + { + "path": "requests/r1/req", + "kind": "unit" + }, + { + "path": "requests/r1/claim", + "kind": "clear" + }, + { + "path": "requests/r1/claim/by", + "kind": "clear" + }, + { + "path": "requests/r1/result", + "kind": "unit" + }, + { + "path": "sessions/s1", + "kind": "struct" + }, + { + "path": "sessions/s1/meta", + "kind": "struct" + }, + { + "path": "sessions/s1/meta/busy", + "kind": "unit" + }, + { + "path": "sessions/s1/meta/busy/phase", + "kind": "below" + }, + { + "path": "sessions/s1/msgs/m0001", + "kind": "unit" + }, + { + "path": "sessions/s1/receipts/r0001", + "kind": "unit" + }, + { + "path": "sessions/s1/user", + "kind": "struct" + }, + { + "path": "sessions/s1/user/archived", + "kind": "unit" + }, + { + "path": "sessions/s1/progress", + "kind": "struct" + }, + { + "path": "sessions/s1/progress/rid", + "kind": "clear" + }, + { + "path": "sessions/s1/progress/steps/s0001", + "kind": "unit" + }, + { + "path": "session_index/s1", + "kind": "unit" + }, + { + "path": "subjects/car/meta", + "kind": "unit" + }, + { + "path": "subjects/car/meta/rev", + "kind": "below" + }, + { + "path": "subjects/car/plan", + "kind": "unit" + }, + { + "path": "subjects/car/cards", + "kind": "unit" + }, + { + "path": "subjects/car/concepts", + "kind": "unit" + }, + { + "path": "log/d1", + "kind": "struct" + }, + { + "path": "log/d1/e1", + "kind": "unit" + }, + { + "path": "feedback/f1", + "kind": "unit" + }, + { + "path": "migration/v1_copied_at", + "kind": "clear" + }, + { + "path": "secrets", + "kind": "unknown" + }, + { + "path": "sessions/s1/other", + "kind": "unknown" + } + ], + "units": [ + { + "path": "requests/01J0000000000000000000000R/req", + "value": { + "kind": "message", + "dev": "d1", + "created": 1800000000000, + "sent_at": 1799999999000, + "body": { + "session": "20261012-170448-5b71", + "text": "zażółć gęślą jaźń" + } + }, + "stored": { + "_e2e": "AVnxpT1g95cKDT1ORKsCeyjnBNJGfjyto/ezodv1ysIc2zHOmj77unCOLXQqYgQjvbfbNub710heso/QkMAUS5Mq+M5+/yIW56HevylofoiyPapWi7obYbu9otQwuOUeiQzutdJpQNw9x7dfUPLZAu1BRQoboIyCG0TrNH3X9LkBIRgWLHOpf3lxhmXGJht3muR4lS4zwDOxj7pB1lRluLM/mnJcXqzDdmgHHw==", + "created": 1800000000000 + } + }, + { + "path": "requests/01J0000000000000000000000R/result", + "value": { + "ok": true, + "at": 1800000000500, + "data": { + "msg": "m0002" + } + }, + "stored": { + "_e2e": "AREDDVq3hgmR9xsL5pUrhSJbK+fBCm823Hx8oKUo+ExwPT342IvIEqu55Rog+u4wWXxON6Vy5xxJZwd4+gXfxkQ49j6cb+H0tyRO", + "at": 1800000000500 + } + }, + { + "path": "presence/clients/d1", + "value": { + "platform": "android", + "app": "1.0.0", + "seen": 1800000000000 + }, + "stored": { + "_e2e": "AYhbgKJJ4tsHmFB0THKTxQoCzwKOpa/3ioNcYmtj2UhmhRZyavA7614RT6y3aPAmBWf5uNhQgkKKOO+Qavw4XAad+7xttA23RB7Xhb0=", + "seen": 1800000000000 + } + }, + { + "path": "prefs/current_subject", + "value": "car", + "stored": { + "_e2e": "AWEy8u/msdxYHsp7PQ0jkzrYpdNP6SEqbJ0SUyqzT2DzoEpXRf+1vKAQVh1hUQ==" + } + }, + { + "path": "sessions/20261012-170448-5b71/user/archived", + "value": true, + "stored": { + "_e2e": "AbS7aTNu+meRs1jvcw/alGgKKgbhV83VPihFMd8AmqW6QhwgCQMg46hnrJls" + } + }, + { + "path": "log/d1/01J0000000000000000000000E", + "value": { + "type": "review", + "card": "ld-x", + "r": 3, + "t": 1800000000000 + }, + "stored": { + "_e2e": "ASqR1ybCH4+WKx7P0nihgr8IOcGMsw1fXXyigH7kgrAZHru4DX0lXW2cZF0DPwQxeay3dS36BYXPN6OgO8xnXjcTz3hUwjjlVLANzZvG0zICImA8rBDy5bInd/INDmc5" + } + } + ], + "header": { + "path": "tutor/e2e", + "passphrase": "correct horse", + "text": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"67H6FMZ+zlSFuBRuMIy2IQ==\",\"m\":8,\"t\":1,\"p\":1,\"check\":\"JQdQLQiowhtNkztQDPtOOUARTzUStpsKZmPHGZXEJ/8=\"}", + "unit": { + "path": "log/0b6f2c1e-8f2a-4c1d-9e3b-7a5d4c3b2a10/01J0000000000000000000000E", + "value": { + "type": "review", + "card": "ld-x", + "r": 3 + }, + "stored": { + "_e2e": "ATvPPA3qI5F70kbmI3XDguYjUsmQZQD2j8fY85sGGOg5IcxeYQn4mbWichyVU82pQQ7WA1YDngEs0K17HImxMyJSUf2H622XzinCPSqN" + } + } + } +} diff --git a/rtdb_relay/rtdb_relay/tests/test_lease.py b/rtdb_relay/rtdb_relay/tests/test_lease.py new file mode 100644 index 0000000..c90da3e --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/test_lease.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""The claim lease: take, lose a race, renew, lose the lease.""" + +from __future__ import annotations + +import asyncio +import logging + +import pytest + +from rtdb_relay.lease import Lease, claimable, ms, valid_rid +from rtdb_relay.rtdb import ConflictError +from rtdb_relay.tests._fake_rtdb import ROOT, FakeRtdb + +RID = "01J0000000000000000000000R" + + +def test_helpers() -> None: + assert valid_rid(RID) + assert not valid_rid("../x") + assert ms(1.5) == 1500 + assert claimable(None, 0) + assert claimable({"lease_until": 999}, 1.0) + assert not claimable({"lease_until": 2000, "by": "x"}, 1.0) + assert claimable({"lease_until": 2000, "by": "me"}, 1.0, "me") + assert claimable({"lease_until": "?"}, 1.0) + + +def test_take_renew_and_races( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + fake = FakeRtdb() + + async def go() -> None: + db = fake.rtdb() + lease = Lease(db, RID, "pc", lambda: 1.0) + fake.set(f"{ROOT}/{lease.path}", {"attempt": 2, "lease_until": 0}) + assert await lease.take() + assert lease.attempt == 3 + await lease.renew("running") + assert fake.get(f"{ROOT}/{lease.path}")["phase"] == "running" + fake.set(f"{ROOT}/{lease.path}", {"by": "other", "lease_until": 9e15}) + await lease.renew() # lost: only logged + assert fake.get(f"{ROOT}/{lease.path}")["by"] == "other" + assert not await Lease(db, RID, "pc", lambda: 1.0).take() + fake.set(f"{ROOT}/{lease.path}", {"attempt": "x"}) + + async def lose(*args: object) -> tuple[object, str]: + raise ConflictError("p", None, "") + + monkeypatch.setattr(db, "put_etag", lose) + racer = Lease(db, RID, "pc", lambda: 1.0) + assert not await racer.take() + assert racer.attempt == 1 + await db.aclose() + + with caplog.at_level(logging.WARNING): + asyncio.run(go()) + assert "lost to another writer" in caplog.text + assert "claimed" in caplog.text diff --git a/rtdb_relay/rtdb_relay/tests/test_presence.py b/rtdb_relay/rtdb_relay/tests/test_presence.py new file mode 100644 index 0000000..9ca9acc --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/test_presence.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""The worker heartbeat.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from rtdb_relay.presence import beat_forever, node +from rtdb_relay.tests._fake_rtdb import ROOT, FakeRtdb + + +def test_beats_until_cancelled() -> None: + fake = FakeRtdb() + sleeps: list[float] = [] + + async def go() -> None: + db = fake.rtdb() + + async def sleep(s: float) -> None: + sleeps.append(s) + if len(sleeps) == 2: + raise asyncio.CancelledError + + with pytest.raises(asyncio.CancelledError): + await beat_forever(db, "pc", "1.0", sleep) + await db.aclose() + + asyncio.run(go()) + stored = fake.get(f"{ROOT}/presence/workers/pc") + assert stored["beat"] == fake.now_ms + assert stored["version"] == "1.0" + assert sleeps == [30.0, 30.0] + assert set(node("v")) == {"beat", "host", "version"} diff --git a/rtdb_relay/rtdb_relay/tests/test_rtdb.py b/rtdb_relay/rtdb_relay/tests/test_rtdb.py new file mode 100644 index 0000000..e2e73d8 --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/test_rtdb.py @@ -0,0 +1,163 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""The RTDB REST client against the in-memory fake: ETags, 412, errors.""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING, Any + +import httpx +import pytest + +from rtdb_relay import rtdb as rtdb_mod +from rtdb_relay.rtdb import ConflictError, Rtdb, RtdbError +from rtdb_relay.tests._fake_rtdb import DB, ROOT, FakeRtdb, etag + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Coroutine + + +async def _drain(db: Rtdb) -> list[str]: + return [line async for line in db.open_stream("s")] + + +def run(coro: Coroutine[Any, Any, None]) -> None: + asyncio.run(coro) + + +def test_url_quotes_segments_and_root() -> None: + fake = FakeRtdb(sealed=False) + db = fake.rtdb() + assert db.url("requests/a b") == f"{DB}/tutor/v1-e2e/requests/a%20b.json" + assert db.url("/") == f"{DB}/tutor/v1-e2e.json" + assert db.url("subjects/pojęcia-z-łodzi") == ( + f"{DB}/tutor/v1-e2e/subjects/poj%C4%99cia-z-%C5%82odzi.json" + ) + run(db.aclose()) + + +def test_put_get_patch_delete_round_trip() -> None: + fake = FakeRtdb(now_ms=42, sealed=False) + + async def go() -> None: + db = fake.rtdb() + stored = await db.put("a/b", {"x": 1, "at": rtdb_mod.SERVER_TIME}) + assert stored == {"x": 1, "at": 42} + await db.patch("a/b", {"y": 2, "x": None}) + assert await db.get("a/b") == {"at": 42, "y": 2} + assert await db.get("a", {"shallow": "true"}) == {"b": True} + await db.delete("a/b") + assert await db.get("a/b") is None + await db.aclose() + + run(go()) + + +def test_conditional_put_and_lost_race() -> None: + fake = FakeRtdb(sealed=False) + + async def go() -> None: + db = fake.rtdb() + value, tag = await db.get_etag("c") + assert value is None + assert tag == "null_etag" + _, tag2 = await db.put_etag("c", {"v": 1}, tag) + assert tag2 == etag({"v": 1}) + with pytest.raises(ConflictError) as lost: + await db.put_etag("c", {"v": 2}, tag) + assert lost.value.value == {"v": 1} + assert lost.value.etag == tag2 + await db.delete("c", if_match=tag2) + await db.aclose() + + run(go()) + + +def test_http_errors_never_show_the_url_or_token( + caplog: pytest.LogCaptureFixture, +) -> None: + fake = FakeRtdb(sealed=False) + fake.fail[("GET", "/x")] = 500 + fake.raise_on.add(("PUT", "/y")) + + async def go() -> None: + db = fake.rtdb() + with pytest.raises(RtdbError, match=r"^GET x: HTTP 500$"): + await db.get("x") + with pytest.raises(RtdbError, match=r"^PUT y: ConnectError$"): + await db.put("y", 1) + await db.aclose() + + with caplog.at_level(logging.DEBUG): + run(go()) + assert "test-token" not in caplog.text + + +def test_wrong_token_is_an_error() -> None: + fake = FakeRtdb(sealed=False) + fake.token = fake.token[::-1] + + async def go() -> None: + db = fake.rtdb() + with pytest.raises(RtdbError, match="HTTP 401"): + await db.get("x") + await db.aclose() + + run(go()) + + +def test_default_client_is_created() -> None: + async def token() -> str: + return "t" + + async def go() -> None: + db = Rtdb(DB + "/", token, root=ROOT) + assert isinstance(db.client, httpx.AsyncClient) + assert db.url("x") == f"{DB}/tutor/v1-e2e/x.json" + with pytest.raises(RtdbError, match="GET x: "): + await db.get("x") # the test sandbox blocks real network + await db.aclose() + + run(go()) + + +def test_open_stream_errors() -> None: + fake = FakeRtdb(sealed=False) + fake.fail[("GET", "/s")] = 401 + fake.raise_on.add(("GET", "/t")) + + async def drain(db: Rtdb, path: str) -> list[str]: + return [line async for line in db.open_stream(path)] + + async def go() -> None: + db = fake.rtdb() + with pytest.raises(RtdbError, match="STREAM s: HTTP 401"): + await drain(db, "s") + with pytest.raises(RtdbError, match="STREAM t: ConnectError"): + await drain(db, "t") + await db.aclose() + + run(go()) + + +def test_open_stream_read_error_mid_stream() -> None: + async def handler(request: httpx.Request) -> httpx.Response: + async def body() -> AsyncIterator[bytes]: + yield b"event: keep-alive\n" + msg = "cut" + raise httpx.ReadError(msg, request=request) + + return httpx.Response(200, content=body()) + + async def token() -> str: + return "t" + + async def go() -> None: + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + db = Rtdb(DB, token, client=client, root=ROOT) + with pytest.raises(RtdbError, match="STREAM s: ReadError"): + await _drain(db) + await db.aclose() + + run(go()) diff --git a/rtdb_relay/rtdb_relay/tests/test_seal.py b/rtdb_relay/rtdb_relay/tests/test_seal.py new file mode 100644 index 0000000..97fa143 --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/test_seal.py @@ -0,0 +1,112 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Sealing: the tutor's contract vectors still open and classify the same.""" + +from __future__ import annotations + +import asyncio +import json + +from crdt_sync import E2eCipher +import pytest + +from rtdb_relay import sse +from rtdb_relay.layout import E2eLayoutError, Layout, parts_of +from rtdb_relay.seal import SEALED, Sealer, dumps +from rtdb_relay.tests._fake_rtdb import ROOT, FakeRtdb +from rtdb_relay.tests._layouts import TUTOR, VECTORS + + +def _sealer() -> Sealer: + return Sealer(E2eCipher(bytes.fromhex(VECTORS["key_hex"])), TUTOR) + + +def test_vectors_pin_the_rules_and_open() -> None: + assert VECTORS["root"] == TUTOR.root == "tutor/v1-e2e" + assert VECTORS["sealed_key"] == SEALED + for rule in VECTORS["rules"]: + assert TUTOR.classify(parts_of(rule["path"])).kind == rule["kind"], rule + s = _sealer() + for unit in VECTORS["units"]: + assert s.open(unit["path"], unit["stored"]) == unit["value"] + assert set(unit["stored"]) - {SEALED} <= {"created", "at", "seen", "beat"} + + +def test_sealed_bytes_are_the_cipher_over_compact_json_at_the_full_path() -> None: + """Differential: our seal == crdt_sync's cipher on the vectors' format.""" + cipher = E2eCipher(bytes.fromhex(VECTORS["key_hex"])) + s = Sealer(cipher, TUTOR) + for unit in VECTORS["units"]: + ours = s.seal(unit["path"], unit["value"]) + assert isinstance(ours, dict) + assert isinstance(unit["stored"], dict) + clear = {k: v for k, v in unit["stored"].items() if k != SEALED} + assert {k: v for k, v in ours.items() if k != SEALED} == clear + aad = f"{TUTOR.root}/{unit['path']}" + text = cipher.open(ours[SEALED], path=aad) + value = unit["value"] + rest = ( + {k: v for k, v in value.items() if k not in clear} + if isinstance(value, dict) + else value + ) + assert text == dumps(rest) + # And the vector's own blob opens to the same plaintext bytes. + assert cipher.open(unit["stored"][SEALED], path=aad) == text + + +def test_seal_refuses_unknown_and_inside_unit_writes() -> None: + s = _sealer() + with pytest.raises(E2eLayoutError): + s.seal("secrets", {"a": 1}) + with pytest.raises(E2eLayoutError): + s.seal("sessions", "flat") # a scalar where a tree belongs + with pytest.raises(E2eLayoutError): + s.seal_patch("sessions/s/msgs/m0001", {"text": "x"}) + with pytest.raises(E2eLayoutError): + s.open("stats/today/x", 1) + assert s.unit_path("stats/today/x") == "stats/today" + assert s.unit_path("stats/today") is None + assert s.seal("live", {"a": None}) is None # pruned to nothing + assert s.seal("meta/x", 1) == 1 + assert s.seal_patch("prefs", {"a": None}) == {"a": None} + + +def test_open_drops_a_moved_or_tampered_unit_and_keeps_plaintext() -> None: + s = _sealer() + sealed = s.seal("prefs/current_subject", "car") + assert s.open("prefs", {"current_subject": sealed}) == {"current_subject": "car"} + assert s.open("prefs", {"other": sealed}) == {} + assert s.open("prefs", {"current_subject": {SEALED: "AQ=="}}) == {} + assert s.open("prefs/current_subject", "car") == "car" + assert s.open("sessions", [{"x": 1}]) == [{"x": 1}] + assert s.open("meta", {"a": 1}) == {"a": 1} + + +def test_root_override_changes_the_aad() -> None: + layout = Layout(root="app/v1-e2e", clear=(), units={"x": frozenset()}) + cipher = E2eCipher(bytes(range(32))) + blob = Sealer(cipher, layout).seal("x", 1) + assert Sealer(cipher, layout, root="other").open("x", blob) is None + assert Sealer(cipher, layout).open("x", blob) == 1 + + +def test_rtdb_reads_below_a_unit_and_streams_open() -> None: + fake = FakeRtdb() + fake.set(f"{ROOT}/stats/today", {"new_cap": 25}) + + async def go() -> None: + db = fake.rtdb() + assert await db.get("stats/today/new_cap") == 25 + assert await db.get("stats/today/nope/deeper") is None + stream = sse.listen(db, "stats") + first = await anext(stream) + assert first.data == {"today": {"new_cap": 25}} + await db.patch("stats", {"days": {"2026-11-12": {"m": 3}}}) + second = await anext(stream) + assert second.kind == "patch" + assert second.data == {"days": {"2026-11-12": {"m": 3}}} + await stream.aclose() + await db.aclose() + + asyncio.run(go()) + assert "new_cap" not in json.dumps(fake.raw) diff --git a/rtdb_relay/rtdb_relay/tests/test_server.py b/rtdb_relay/rtdb_relay/tests/test_server.py new file mode 100644 index 0000000..80b0da7 --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/test_server.py @@ -0,0 +1,147 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""The request server: claim, run the kind's handler, write the result.""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +import pytest + +from rtdb_relay import server as server_mod +from rtdb_relay.server import RequestError, RequestServer +from rtdb_relay.tests._fake_rtdb import ROOT, FakeRtdb + +RID = "01J0000000000000000000000R" +NOW = 1_800_000_000.0 + + +async def _echo(body: dict[str, Any]) -> dict[str, Any]: + return {"echo": body.get("x")} + + +async def _refuse(body: dict[str, Any]) -> dict[str, Any]: + raise RequestError("not_now", "cannot start", retry=True) + + +async def _bug(body: dict[str, Any]) -> dict[str, Any]: + raise KeyError(body["missing"]) + + +HANDLERS = {"echo": _echo, "refuse": _refuse, "bug": _bug} + + +def _req(kind: str, body: object = None) -> dict[str, Any]: + return {"kind": kind, "dev": "d1", "created": 1, "sent_at": 1, "body": body} + + +def _answer(fake: FakeRtdb, rid: str = RID) -> dict[str, Any] | None: + async def go() -> dict[str, Any] | None: + db = fake.rtdb() + srv = RequestServer(db, "pc", HANDLERS, now=lambda: NOW) + node = await srv._raw.get(f"requests/{rid}") + out = await srv.answer(rid, node if isinstance(node, dict) else {}) + await db.aclose() + return out + + return asyncio.run(go()) + + +@pytest.mark.parametrize( + ("kind", "body", "want"), + [ + ("echo", {"x": 7}, {"ok": True, "kind": "echo", "echo": 7}), + ("echo", "not a dict", {"ok": True, "kind": "echo"}), + ("nope", None, {"ok": False, "kind": "nope", "code": "unknown_kind"}), + ("refuse", {}, {"ok": False, "code": "not_now", "retry": True}), + ("bug", {}, {"ok": False, "code": "internal", "error": "KeyError"}), + ], +) +def test_answers_by_kind(kind: str, body: object, want: dict[str, Any]) -> None: + fake = FakeRtdb() + fake.set(f"{ROOT}/requests/{RID}/req", _req(kind, body)) + assert _answer(fake) is not None + result = fake.get(f"{ROOT}/requests/{RID}/result") + assert want.items() <= result.items() + assert result["at"] == fake.now_ms + claim = fake.get(f"{ROOT}/requests/{RID}/claim") + assert claim["by"] == "pc" + assert claim["attempt"] == 1 + + +def test_refuses_an_unsealed_request_and_skips_what_is_not_pending() -> None: + fake = FakeRtdb() + fake.raw_set(f"{ROOT}/requests/{RID}/req", _req("echo", {"x": 1})) + assert _answer(fake) is not None + assert fake.get(f"{ROOT}/requests/{RID}/result")["code"] == "unsealed" + assert _answer(fake) is None # already has a result + assert _answer(fake, "not-a-ulid") is None + fake2 = FakeRtdb() + fake2.set(f"{ROOT}/requests/{RID}/claim", {"by": "other", "lease_until": 9e15}) + assert _answer(fake2) is None # no req + fake2.set(f"{ROOT}/requests/{RID}/req", _req("echo")) + assert _answer(fake2) is None # another worker holds the lease + + +def test_wrong_key_request_reads_as_unsealed() -> None: + fake = FakeRtdb() + fake.raw_set(f"{ROOT}/requests/{RID}/req", {"_e2e": "AQ==", "created": 1}) + _answer(fake) + assert fake.get(f"{ROOT}/requests/{RID}/result")["code"] == "unsealed" + + +def test_run_serves_new_requests_and_renews_long_leases( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(server_mod, "RENEW_S", 0.01) + fake = FakeRtdb() + seen: list[str] = [] + + async def slow(body: dict[str, Any]) -> dict[str, Any]: + await asyncio.sleep(0.05) + claim = fake.get(f"{ROOT}/requests/{RID}/claim") + seen.append(claim["phase"]) + return {} + + async def go() -> None: + db = fake.rtdb() + srv = RequestServer(db, "pc", {"slow": slow}) + task = asyncio.create_task(srv.run()) + await asyncio.sleep(0.01) + await db.put(f"requests/{RID}/req", _req("slow")) + for _ in range(200): + if fake.get(f"{ROOT}/requests/{RID}/result"): + break + await asyncio.sleep(0.01) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await db.aclose() + + asyncio.run(go()) + assert fake.get(f"{ROOT}/requests/{RID}/result")["ok"] is True + assert seen == ["running"] + + +def test_dispatch_skips_busy_and_logs_a_failing_request( + caplog: pytest.LogCaptureFixture, +) -> None: + fake = FakeRtdb() + + async def go() -> None: + db = fake.rtdb() + srv = RequestServer(db, "pc", HANDLERS) + + async def boom(rid: str, node: dict[str, Any]) -> None: + raise RuntimeError(rid) + + srv.answer = boom # type: ignore[method-assign] + srv.dispatch({RID: {"req": {}}, "x": "flat", "y": {"result": {}}}) + srv.dispatch({RID: {"req": {}}}) # in flight: not started twice + assert len(srv.tasks) == 1 + await asyncio.gather(*srv.tasks) + await db.aclose() + + with caplog.at_level(logging.ERROR): + asyncio.run(go()) + assert f"request {RID} failed" in caplog.text diff --git a/rtdb_relay/rtdb_relay/tests/test_sse.py b/rtdb_relay/rtdb_relay/tests/test_sse.py new file mode 100644 index 0000000..0cf2621 --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/test_sse.py @@ -0,0 +1,145 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""SSE parsing, reconnects (drop, auth_revoked, errors), cancel; the mirror.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +import pytest + +from rtdb_relay import mirror, sse +from rtdb_relay.sse import Event, StreamCancelledError, parse +from rtdb_relay.tests._fake_rtdb import FakeRtdb + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +def test_parse_block() -> None: + assert parse(["event: put", 'data: {"a":1}', "id: 3"]) == ("put", '{"a":1}') + assert parse(["data: x", "data: y"]) == ("message", "x\ny") + + +async def _take(agen: AsyncIterator[Event], n: int) -> list[Event]: + out = [] + async for event in agen: + out.append(event) + if len(out) == n: + break + return out + + +def test_listen_snapshot_then_writes_and_reconnects() -> None: + fake = FakeRtdb() + fake.set("tutor/v1-e2e/requests/a", {"req": 1}) + sleeps: list[float] = [] + + async def sleep(delay: float) -> None: + sleeps.append(delay) + + async def go() -> list[Event]: + db = fake.rtdb() + stream = sse.listen(db, "requests", sleep=sleep) + got = await _take(stream, 1) + await db.put("requests/b/req", 2) + got += await _take(stream, 1) + fake.push("event: keep-alive\ndata: null\n\n") + fake.push("event: auth_revoked\ndata: credential is no longer valid\n\n") + got += await _take(stream, 1) # reconnected: snapshot again + fake.push(None) # server closes: reconnect at once + got += await _take(stream, 1) + fake.raise_on.add(("GET", "/requests")) + fake.push(None) + + async def heal(delay: float) -> None: + sleeps.append(delay) + fake.raise_on.clear() + + stream2 = sse.listen(db, "requests", sleep=heal) + fake.raise_on.add(("GET", "/requests")) + got += await _take(stream2, 1) + fake.push('event: put\ndata: {"path": "/x", "data": 1}\n\n') + got += await _take(stream2, 1) + await stream.aclose() + await stream2.aclose() + await db.aclose() + return got + + got = asyncio.run(go()) + assert got[0] == Event("put", "/", {"a": {"req": 1}}) + assert got[1] == Event("put", "/b/req", 2) + assert got[2].path == "/" + assert got[2].data == {"a": {"req": 1}, "b": {"req": 2}} + assert got[3].path == "/" + assert got[4].path == "/" + assert got[5] == Event("put", "/x", 1) + assert sleeps == [1.0] + + +def test_listen_backoff_grows_and_bad_json_retries() -> None: + fake = FakeRtdb() + fake.raise_on.add(("GET", "/n")) + sleeps: list[float] = [] + + async def sleep(delay: float) -> None: + sleeps.append(delay) + if len(sleeps) == 6: + fake.raise_on.clear() + + async def go() -> list[Event]: + db = fake.rtdb() + stream = sse.listen(db, "n", sleep=sleep) + got = await _take(stream, 1) + fake.push("event: patch\ndata: {not json\n\n") + got += await _take(stream, 1) # dropped, backed off, reconnected + await stream.aclose() + await db.aclose() + return got + + got = asyncio.run(go()) + assert [e.path for e in got] == ["/", "/"] + assert sleeps == [1.0, 2.0, 5.0, 10.0, 30.0, 30.0, 1.0] + + +def test_cancel_is_fatal() -> None: + fake = FakeRtdb() + + async def go() -> None: + db = fake.rtdb() + stream = sse.listen(db, "c") + await _take(stream, 1) + fake.push("event: cancel\ndata: permission denied\n\n") + with pytest.raises(StreamCancelledError): + await _take(stream, 1) + await db.aclose() + + asyncio.run(go()) + + +def test_mirror_applies_put_and_patch() -> None: + tree = mirror.apply({}, Event("put", "/", {"a": {"req": 1}})) + tree = mirror.apply(tree, Event("put", "/b/req", 2)) + tree = mirror.apply(tree, Event("patch", "/a", {"claim": {"x": 1}, "req": None})) + assert tree == {"a": {"claim": {"x": 1}}, "b": {"req": 2}} + tree = mirror.apply(tree, Event("put", "/a/claim/x", None)) + assert tree == {"b": {"req": 2}} + tree = mirror.apply(tree, Event("put", "/b/req/deep", 3)) # replaces a leaf + assert tree == {"b": {"req": {"deep": 3}}} + tree = mirror.apply(tree, Event("patch", "/", {"c/d": 4})) + assert tree["c"] == {"d": 4} + tree = mirror.apply(tree, Event("patch", "/", None)) + assert mirror.apply(tree, Event("put", "/", None)) == {} + assert mirror.first_key("/a/b") == "a" + assert mirror.first_key("/") is None + + +def test_blank_lines_without_a_block_are_skipped() -> None: + async def lines() -> AsyncIterator[str]: + for line in ("", "event: put", "data: {}", "", ""): + yield line + + async def go() -> list[tuple[str, str]]: + return [b async for b in sse._blocks(lines())] + + assert asyncio.run(go()) == [("put", "{}")] diff --git a/rtdb_relay/rtdb_relay/tests/test_unlock.py b/rtdb_relay/rtdb_relay/tests/test_unlock.py new file mode 100644 index 0000000..f546d1e --- /dev/null +++ b/rtdb_relay/rtdb_relay/tests/test_unlock.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""Unlocking: header made once, wrong or missing passphrase refused.""" + +from __future__ import annotations + +import asyncio + +from crdt_sync import Argon2Params, E2eLayout, E2eLockedError +import pytest + +from rtdb_relay.tests._fake_rtdb import FakeRtdb +from rtdb_relay.unlock import HeaderStore, unlock + +FAST = Argon2Params(memory_kib=8, iterations=1, parallelism=1) +LAYOUT = E2eLayout.of("tutor/v1") + + +def test_unlock_creates_the_header_then_refuses_a_wrong_passphrase() -> None: + fake = FakeRtdb(sealed=False) + + async def go() -> None: + db = fake.rtdb() + key = await unlock(db, "pw", LAYOUT, FAST) + header = fake.raw_get(LAYOUT.e2e_path) + assert isinstance(header, str) + assert "argon2id" in header + again = await unlock(db, "pw", LAYOUT, FAST) + assert again.master == key.master + with pytest.raises(E2eLockedError): + await unlock(db, "other", LAYOUT, FAST) + with pytest.raises(E2eLockedError): + await unlock(db, "", LAYOUT, FAST) + fake.raw_set(LAYOUT.e2e_path, {"not": "a header"}) + with pytest.raises(E2eLockedError): + await unlock(db, "pw", LAYOUT, FAST) + await db.aclose() + + asyncio.run(go()) + + +def test_header_store_lists_nothing_and_never_deletes() -> None: + fake = FakeRtdb(sealed=False) + + async def go() -> None: + db = fake.rtdb() + store = HeaderStore(db.at(""), asyncio.get_running_loop()) + assert store.list_directory("x") == [] + assert store.can_access_remote() + with pytest.raises(NotImplementedError): + store.delete_file("x") + await db.aclose() + + asyncio.run(go()) diff --git a/rtdb_relay/rtdb_relay/unlock.py b/rtdb_relay/rtdb_relay/unlock.py new file mode 100644 index 0000000..2db8be9 --- /dev/null +++ b/rtdb_relay/rtdb_relay/unlock.py @@ -0,0 +1,97 @@ +# Copyright (c) 2026 Krzysztof Rudnicki. MIT License. +"""The E2E key from the shared crdt-sync passphrase (header made on first use).""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any + +from crdt_sync import ( + Argon2Params, + E2eHeaderError, + E2eKey, + E2eLayout, + E2eLockedError, + E2eWrongPassphraseError, + unlock_e2e, +) + +from rtdb_relay.seal import dumps + +if TYPE_CHECKING: + from collections.abc import Coroutine + + from rtdb_relay.rtdb import Rtdb + + +class HeaderStore: + """crdt_sync's sync ``RemoteStore``, just enough for ``unlock_e2e``. + + ``unlock_e2e`` runs in a worker thread (Argon2id must not block the + loop); each call hops back to the loop that owns the async client. + The header is one JSON string leaf, as the Flutter side writes it. + """ + + def __init__(self, raw: Rtdb, loop: asyncio.AbstractEventLoop) -> None: + """Read and write through ``raw`` (an unsealed client on ``""``).""" + self._raw = raw + self._loop = loop + + def _run[T](self, coro: Coroutine[Any, Any, T]) -> T: + return asyncio.run_coroutine_threadsafe(coro, self._loop).result() + + def list_directory(self, path: str) -> list[str]: + """Nothing: only the header is read through here.""" + del path + return [] + + def get_file_text(self, path: str) -> str | None: + """The header text, or ``None``; a non-string value comes back as JSON.""" + value = self._run(self._raw.get(path)) + if value is None or isinstance(value, str): + return value + return dumps(value) # not ours: E2eHeaderError, never overwritten + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Write the header as one string leaf.""" + del message + self._run(self._raw.put(path, text)) + + def delete_file(self, path: str, *, message: str = "") -> None: + """Refused: the header is never deleted.""" + del path, message + msg = "the relay never deletes the e2e header" + raise NotImplementedError(msg) + + def can_access_remote(self) -> bool: + """Always: a failing call raises on its own.""" + return True + + +async def unlock( + rtdb: Rtdb, + passphrase: str, + layout: E2eLayout, + params: Argon2Params | None = None, +) -> E2eKey: + """The key of ``layout`` (``E2eLayout.of("/v1")``) from ``passphrase``. + + Raises: + E2eLockedError: No passphrase, a wrong one, or an unreadable header. + + """ + if not passphrase: + msg = "no crdt-sync passphrase on this PC" + raise E2eLockedError(msg) + store = HeaderStore(rtdb.at(""), asyncio.get_running_loop()) + try: + return await asyncio.to_thread( + unlock_e2e, + store, + path_prefix=layout.prefix, + passphrase=passphrase, + params=params, + e2e_path=layout.e2e_path, + ) + except (E2eWrongPassphraseError, E2eHeaderError) as exc: + raise E2eLockedError(str(exc) or type(exc).__name__) from exc