From 47f2d239c44525cd2dabd7c1574f399f543a5714 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Thu, 12 Nov 2026 09:47:33 +0100 Subject: [PATCH] crdt-sync 0.10.0: E2E port (Argon2id, XChaCha20) Byte-compatible with crdt_sync_dart 0.14.0, proven by golden vectors in both directions. EncryptingStore, -e2e layout, one-time copy, and open_encrypted_client for the Python daemons. Dev pins bumped. Todo: fda71ed1 Co-Authored-By: Claude Opus 5.5 --- crdt-sync/.pre-commit-config.yaml | 2 +- crdt-sync/crdt_sync/__init__.py | 40 ++++ crdt-sync/crdt_sync/_e2e_cipher.py | 202 +++++++++++++++++ crdt-sync/crdt_sync/_e2e_key.py | 191 ++++++++++++++++ crdt-sync/crdt_sync/_e2e_store.py | 210 ++++++++++++++++++ crdt-sync/crdt_sync/_pull.py | 14 +- .../tests/fixtures/e2e_dart_vectors.json | 10 + crdt-sync/crdt_sync/tests/test_e2e_cipher.py | 146 ++++++++++++ crdt-sync/crdt_sync/tests/test_e2e_interop.py | 74 ++++++ crdt-sync/crdt_sync/tests/test_e2e_key.py | 123 ++++++++++ crdt-sync/crdt_sync/tests/test_e2e_store.py | 118 ++++++++++ .../crdt_sync/tests/test_e2e_store_edges.py | 165 ++++++++++++++ .../crdt_sync/tests/test_sync_e2e_skip.py | 32 +++ .../crdt_sync/tests/test_tool_e2e_vectors.py | 41 ++++ crdt-sync/pyproject.toml | 8 +- crdt-sync/requirements.txt | 10 +- crdt-sync/tool/e2e_vectors.py | 64 ++++++ 17 files changed, 1440 insertions(+), 10 deletions(-) create mode 100644 crdt-sync/crdt_sync/_e2e_cipher.py create mode 100644 crdt-sync/crdt_sync/_e2e_key.py create mode 100644 crdt-sync/crdt_sync/_e2e_store.py create mode 100644 crdt-sync/crdt_sync/tests/fixtures/e2e_dart_vectors.json create mode 100644 crdt-sync/crdt_sync/tests/test_e2e_cipher.py create mode 100644 crdt-sync/crdt_sync/tests/test_e2e_interop.py create mode 100644 crdt-sync/crdt_sync/tests/test_e2e_key.py create mode 100644 crdt-sync/crdt_sync/tests/test_e2e_store.py create mode 100644 crdt-sync/crdt_sync/tests/test_e2e_store_edges.py create mode 100644 crdt-sync/crdt_sync/tests/test_sync_e2e_skip.py create mode 100644 crdt-sync/crdt_sync/tests/test_tool_e2e_vectors.py create mode 100644 crdt-sync/tool/e2e_vectors.py diff --git a/crdt-sync/.pre-commit-config.yaml b/crdt-sync/.pre-commit-config.yaml index f4a3ac5..2015dfc 100644 --- a/crdt-sync/.pre-commit-config.yaml +++ b/crdt-sync/.pre-commit-config.yaml @@ -65,7 +65,7 @@ repos: types: [python] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.17.2 + rev: v0.17.4 hooks: - id: ruff args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix, --show-fixes] diff --git a/crdt-sync/crdt_sync/__init__.py b/crdt-sync/crdt_sync/__init__.py index 61526aa..d614d2d 100644 --- a/crdt-sync/crdt_sync/__init__.py +++ b/crdt-sync/crdt_sync/__init__.py @@ -14,6 +14,29 @@ from crdt_sync._config import ( mirror_client_for, ) from crdt_sync._device import DeviceIdentity, load_device_identity +from crdt_sync._e2e_cipher import ( + E2eCipher, + E2eDecryptError, + is_e2e_envelope, + open_log_text, + seal_log_text, +) +from crdt_sync._e2e_key import ( + Argon2Params, + E2eHeaderError, + E2eKey, + E2eWrongPassphraseError, + default_e2e_path, + unlock_e2e, +) +from crdt_sync._e2e_store import ( + E2eLayout, + E2eLockedError, + EncryptingStore, + copy_to_encrypted, + open_encrypted_client, + reset_e2e_copy_memo, +) from crdt_sync._firebase import ( DatabaseNotFoundError, FirebaseSyncClient, @@ -51,10 +74,19 @@ __all__ = [ "CONFIG_DIR", "CONFIG_FILE", "PASSWORD_FILE", + "Argon2Params", "ConfigError", "CredentialStore", "DatabaseNotFoundError", "DeviceIdentity", + "E2eCipher", + "E2eDecryptError", + "E2eHeaderError", + "E2eKey", + "E2eLayout", + "E2eLockedError", + "E2eWrongPassphraseError", + "EncryptingStore", "Field", "FileCredentialStore", "FileSyncStateStore", @@ -81,18 +113,26 @@ __all__ = [ "SyncState", "SyncStateStore", "SyncTarget", + "copy_to_encrypted", "credential_store_for", + "default_e2e_path", "default_revs_path", "dump_log", "firebase_client_for", + "is_e2e_envelope", "load_device_identity", "load_log", "merge_field", "merge_logs", "merge_record", "mirror_client_for", + "open_encrypted_client", + "open_log_text", "read_log", + "reset_e2e_copy_memo", "revision_of", + "seal_log_text", "sync_log", + "unlock_e2e", "write_log", ] diff --git a/crdt-sync/crdt_sync/_e2e_cipher.py b/crdt-sync/crdt_sync/_e2e_cipher.py new file mode 100644 index 0000000..b8f801b --- /dev/null +++ b/crdt-sync/crdt_sync/_e2e_cipher.py @@ -0,0 +1,202 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""XChaCha20-Poly1305 blob cipher and per-record envelope. + +Byte-compatible with ``crdt_sync_dart``'s ``E2eCipher`` and +``sealLogText``/``openLogText``: a blob is base64 ``[version][nonce][ct][tag]`` +with ``[version] + utf8(path)`` as AAD; an envelope keeps record ids, the +tombstone and the newest HLC clear and seals each record's fields. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import secrets +from typing import cast + +from nacl.bindings import ( + crypto_aead_xchacha20poly1305_ietf_decrypt, + crypto_aead_xchacha20poly1305_ietf_encrypt, +) +from nacl.exceptions import CryptoError + +from crdt_sync._hlc import Hlc + +E2E_FORMAT_VERSION = 1 +_NONCE = 24 +_TAG = 16 +_KEY = 32 + + +class E2eDecryptError(ValueError): + """A blob that is tampered, under another key, or in an unknown format. + + A ``ValueError`` so the pull loop skips the peer instead of aborting. + """ + + +def is_plaintext(stored: str) -> bool: + """Return whether ``stored`` is legacy JSON rather than a sealed blob.""" + return stored.lstrip().startswith(("{", "[")) + + +def _aad(version: int, path: str) -> bytes: + return bytes([version]) + path.encode() + + +class E2eCipher: + """Seals and opens blobs bound to their remote path.""" + + def __init__(self, key: bytes, *, accept_plaintext: bool = True) -> None: + """Wrap a 32-byte ``key``; ``accept_plaintext`` reads old peers.""" + if len(key) != _KEY: + msg = "key must be 32 bytes" + raise ValueError(msg) + self._key = bytes(key) + self.accept_plaintext = accept_plaintext + + def seal(self, plaintext: str, *, path: str) -> str: + """Return ``plaintext`` sealed for ``path``.""" + nonce = secrets.token_bytes(_NONCE) + box = crypto_aead_xchacha20poly1305_ietf_encrypt( + plaintext.encode(), _aad(E2E_FORMAT_VERSION, path), nonce, self._key + ) + return base64.b64encode(bytes([E2E_FORMAT_VERSION]) + nonce + box).decode() + + def open(self, stored: str, *, path: str) -> str: + """Return the plaintext of ``stored``; raise :class:`E2eDecryptError`.""" + if is_plaintext(stored): + if self.accept_plaintext: + return stored + msg = f"plaintext blob at {path} after migration" + raise E2eDecryptError(msg) + try: + raw = base64.b64decode(stored, validate=True) + except binascii.Error as exc: + msg = f"blob at {path} is neither JSON nor base64" + raise E2eDecryptError(msg) from exc + if not raw: + msg = f"empty blob at {path}" + raise E2eDecryptError(msg) + if raw[0] != E2E_FORMAT_VERSION: + msg = f"blob at {path} uses format {raw[0]}; update the app" + raise E2eDecryptError(msg) + if len(raw) < 1 + _NONCE + _TAG: + msg = f"blob at {path} is truncated" + raise E2eDecryptError(msg) + try: + clear = crypto_aead_xchacha20poly1305_ietf_decrypt( + raw[1 + _NONCE :], _aad(raw[0], path), raw[1 : 1 + _NONCE], self._key + ) + except CryptoError as exc: + msg = f"blob at {path} failed authentication" + raise E2eDecryptError(msg) from exc + return clear.decode() + + +def is_e2e_envelope(text: str) -> bool: + """Return whether ``text`` is an envelope (integer ``"v"``).""" + try: + data = json.loads(text) + except ValueError: + return False + version = data.get("v") if isinstance(data, dict) else None + return isinstance(version, int) and not isinstance(version, bool) + + +def _record_aad( + path: str, rid: str, deleted: object, deleted_hlc: object, newest: object +) -> str: + flag = 1 if deleted is True else 0 + return f"{path}#{rid}#{flag}#{deleted_hlc or ''}#{newest or ''}" + + +def _as_log(text: str) -> dict[str, dict[str, object]] | None: + try: + raw = json.loads(text) + except ValueError: + return None + if not isinstance(raw, dict): + return None + for rid, rec in raw.items(): + if not isinstance(rec, dict) or rec.get("id") != rid: + return None + if not isinstance(rec.get("fields"), dict): + return None + return cast("dict[str, dict[str, object]]", raw) + + +def _newest(fields: dict[str, list[object]]) -> str | None: + clocks = [Hlc.from_str(str(pair[1])) for pair in fields.values()] + return max(clocks).to_str() if clocks else None + + +def seal_log_text(text: str, cipher: E2eCipher, *, path: str) -> str: + """Seal a log (``dump_log`` shape) as an envelope; anything else whole.""" + log = _as_log(text) + if log is None: + return json.dumps( + {"v": E2E_FORMAT_VERSION, "b": cipher.seal(text, path=path)}, + separators=(",", ":"), + ) + records: dict[str, object] = {} + for rid, rec in log.items(): + fields = cast("dict[str, list[object]]", rec["fields"]) + deleted = rec.get("deleted") is True + deleted_hlc = rec.get("deleted_hlc") + newest = _newest(fields) + records[rid] = { + "d": deleted, + "dh": deleted_hlc, + "h": newest, + "p": cipher.seal( + json.dumps(fields, separators=(",", ":"), ensure_ascii=False), + path=_record_aad(path, rid, deleted, deleted_hlc, newest), + ), + } + return json.dumps( + {"v": E2E_FORMAT_VERSION, "r": records}, + separators=(",", ":"), + ensure_ascii=False, + ) + + +def open_log_text(text: str, cipher: E2eCipher, *, path: str) -> str: + """Open an envelope back to the ``dump_log`` shape.""" + if not is_e2e_envelope(text): + if cipher.accept_plaintext: + return text + msg = f"plaintext blob at {path} after migration" + raise E2eDecryptError(msg) + data = json.loads(text) + if data["v"] != E2E_FORMAT_VERSION: + msg = f"blob at {path} uses format {data['v']}; update the app" + raise E2eDecryptError(msg) + whole = data.get("b") + if isinstance(whole, str): + return cipher.open(whole, path=path) + records = data.get("r") + if not isinstance(records, dict): + msg = f"envelope at {path} is mis-shaped" + raise E2eDecryptError(msg) + out: dict[str, object] = {} + for rid, rec in records.items(): + if ( + not isinstance(rec, dict) + or not isinstance(rec.get("p"), str) + or not isinstance(rec.get("d"), bool) + ): + msg = f"record {rid} at {path} is mis-shaped" + raise E2eDecryptError(msg) + clear = cipher.open( + rec["p"], + path=_record_aad(path, rid, rec["d"], rec.get("dh"), rec.get("h")), + ) + out[rid] = { + "id": rid, + "fields": json.loads(clear), + "deleted": rec["d"], + "deleted_hlc": rec.get("dh"), + } + return json.dumps(out, separators=(",", ":"), ensure_ascii=False) diff --git a/crdt-sync/crdt_sync/_e2e_key.py b/crdt-sync/crdt_sync/_e2e_key.py new file mode 100644 index 0000000..23a3db1 --- /dev/null +++ b/crdt-sync/crdt_sync/_e2e_key.py @@ -0,0 +1,191 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Argon2id key from the shared crdt-sync password, plus HKDF subkeys. + +Byte-compatible with ``crdt_sync_dart``'s ``E2eKey``/``unlockE2e``: the clear +header at ``e2e_path`` holds salt, Argon2id cost and a check MAC, never a key. +""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass +import hashlib +import hmac +import json +import secrets +from typing import TYPE_CHECKING + +from argon2.low_level import Type, hash_secret_raw + +from crdt_sync._e2e_cipher import E2eCipher + +if TYPE_CHECKING: + from crdt_sync._remote import RemoteStore + +_SALT = 16 +_MASTER = 32 +_HEADER_VERSION = 1 + + +@dataclass(frozen=True) +class Argon2Params: + """Argon2id cost, stored in the header so it can be raised later.""" + + memory_kib: int = 19456 + iterations: int = 2 + parallelism: int = 1 + + +class E2eHeaderError(ValueError): + """The header exists but cannot be parsed; it is never overwritten.""" + + +class E2eWrongPassphraseError(Exception): + """The passphrase does not match the header's check.""" + + +def default_e2e_path(path_prefix: str) -> str: + """Return the header path: a sibling of the device dirs, like ``revs``.""" + head, sep, _ = path_prefix.rpartition("/") + return f"{head}/e2e" if sep else f"{path_prefix}/e2e" + + +def _hkdf(master: bytes, label: str) -> bytes: + prk = hmac.new(b"", master, hashlib.sha256).digest() + info = f"crdt_sync e2e {label}".encode() + return hmac.new(prk, info + b"\x01", hashlib.sha256).digest() + + +@dataclass(frozen=True) +class E2eKey: + """A 32-byte master key and its subkeys. Never written to the remote.""" + + master: bytes + enc: bytes + rev: bytes + check: bytes + + @classmethod + def from_bytes(cls, master: bytes) -> E2eKey: + """Expand a cached 32-byte master key.""" + if len(master) != _MASTER: + msg = "master key must be 32 bytes" + raise ValueError(msg) + return cls( + master, _hkdf(master, "enc"), _hkdf(master, "rev"), _hkdf(master, "check") + ) + + def cipher(self, *, accept_plaintext: bool = True) -> E2eCipher: + """Return the blob cipher for this key.""" + return E2eCipher(self.enc, accept_plaintext=accept_plaintext) + + def revision(self, text: str) -> str: + """Return the keyed revision (hex HMAC-SHA256) of ``text``.""" + return hmac.new(self.rev, text.encode(), hashlib.sha256).hexdigest() + + def check_for(self, salt: bytes) -> str: + """Return the header check value for ``salt``.""" + mac = hmac.new(self.check, salt, hashlib.sha256).digest() + return base64.b64encode(mac).decode() + + +@dataclass(frozen=True) +class _Header: + salt: bytes + params: Argon2Params + check: str + + @classmethod + def parse(cls, text: str) -> _Header: + try: + data = json.loads(text) + except ValueError as exc: + msg = "e2e header is not JSON" + raise E2eHeaderError(msg) from exc + if not isinstance(data, dict) or data.get("v") != _HEADER_VERSION: + msg = "e2e header is not a known version" + raise E2eHeaderError(msg) + if data.get("kdf") != "argon2id": + msg = "e2e header is malformed" + raise E2eHeaderError(msg) + try: + return cls( + base64.b64decode(data["salt"]), + Argon2Params(int(data["m"]), int(data["t"]), int(data["p"])), + str(data["check"]), + ) + except (KeyError, TypeError, ValueError) as exc: + msg = "e2e header is malformed" + raise E2eHeaderError(msg) from exc + + def encode(self) -> str: + return json.dumps( + { + "v": _HEADER_VERSION, + "kdf": "argon2id", + "salt": base64.b64encode(self.salt).decode(), + "m": self.params.memory_kib, + "t": self.params.iterations, + "p": self.params.parallelism, + "check": self.check, + }, + separators=(",", ":"), + ) + + +def derive_key(passphrase: str, salt: bytes, params: Argon2Params) -> E2eKey: + """Run Argon2id (v1.3) and expand the subkeys.""" + master = hash_secret_raw( + secret=passphrase.encode(), + salt=salt, + time_cost=params.iterations, + memory_cost=params.memory_kib, + parallelism=params.parallelism, + hash_len=32, + type=Type.ID, + ) + return E2eKey.from_bytes(master) + + +def read_header(store: RemoteStore, path: str) -> _Header | None: + """Return the parsed header at ``path``, or None when absent.""" + text = store.get_file_text(path) + return None if text is None else _Header.parse(text) + + +def key_matches(key: E2eKey, store: RemoteStore, path: str) -> bool: + """Return whether ``key`` matches the header at ``path``.""" + header = read_header(store, path) + return header is not None and hmac.compare_digest( + key.check_for(header.salt), header.check + ) + + +def unlock_e2e( + store: RemoteStore, + *, + path_prefix: str, + passphrase: str, + params: Argon2Params | None = None, + e2e_path: str | None = None, +) -> E2eKey: + """Derive the key from ``passphrase``, creating the header on first use.""" + if not passphrase: + msg = "passphrase must not be empty" + raise ValueError(msg) + path = e2e_path or default_e2e_path(path_prefix) + header = read_header(store, path) + if header is None: + salt = secrets.token_bytes(_SALT) + cost = params or Argon2Params() + key = derive_key(passphrase, salt, cost) + encoded = _Header(salt, cost, key.check_for(salt)).encode() + store.put_file_text(path, encoded, message=path) + stored = store.get_file_text(path) or encoded + if stored == encoded: + return key + header = _Header.parse(stored) + key = derive_key(passphrase, header.salt, header.params) + if not hmac.compare_digest(key.check_for(header.salt), header.check): + raise E2eWrongPassphraseError + return key diff --git a/crdt-sync/crdt_sync/_e2e_store.py b/crdt-sync/crdt_sync/_e2e_store.py new file mode 100644 index 0000000..1f81ae1 --- /dev/null +++ b/crdt-sync/crdt_sync/_e2e_store.py @@ -0,0 +1,210 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Encrypted remote store, ``-e2e`` layout and the one-time copy. + +Mirrors ``crdt_sync_dart``'s ``EncryptingStore``, ``E2eLayout``, +``copyToEncrypted`` and ``openEncryptedStore`` so Python daemons and Flutter +apps share one encrypted prefix per app. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import logging +from typing import TYPE_CHECKING + +from crdt_sync._e2e_cipher import E2eDecryptError, open_log_text, seal_log_text +from crdt_sync._e2e_key import ( + Argon2Params, + E2eHeaderError, + E2eKey, + E2eWrongPassphraseError, + unlock_e2e, +) +from crdt_sync._log import merge_logs +from crdt_sync._store import dump_log, load_log +from crdt_sync._sync import default_revs_path + +if TYPE_CHECKING: + from crdt_sync._remote import RemoteStore + +_logger = logging.getLogger(__name__) + + +class E2eLockedError(Exception): + """No usable key: the caller must not sync at all (fail closed).""" + + +@dataclass(frozen=True) +class E2eLayout: + """Where an app's encrypted data lives, derived from its old prefix.""" + + prefix: str + revs_path: str + e2e_path: str + + @classmethod + def of(cls, old_prefix: str) -> E2eLayout: + """Return ``-e2e`` with revs and header kept out of old paths.""" + prefix = f"{old_prefix}-e2e" + parent, sep, _ = old_prefix.rpartition("/") + if not sep: + return cls(prefix, f"{prefix}/revs", f"{prefix}/e2e") + return cls(prefix, f"{parent}/revs-e2e", f"{parent}/e2e") + + +class EncryptingStore: + """Seals log blobs under the layout's prefix and keys its revisions.""" + + def __init__(self, inner: RemoteStore, key: E2eKey, layout: E2eLayout) -> None: + """Wrap ``inner``; other paths pass through unchanged.""" + self._inner = inner + self.key = key + self._layout = layout + self._cipher = key.cipher() + + def _is_rev(self, path: str) -> bool: + return path.startswith(f"{self._layout.revs_path}/") + + def _is_log(self, path: str) -> bool: + return ( + not self._is_rev(path) + and path != self._layout.e2e_path + and path.startswith(f"{self._layout.prefix}/") + ) + + def list_directory(self, path: str) -> list[str]: + """Delegate.""" + return self._inner.list_directory(path) + + def get_file_text(self, path: str) -> str | None: + """Read and open a sealed log; other paths pass through.""" + text = self._inner.get_file_text(path) + if text is None or not self._is_log(path): + return text + return open_log_text(text, self._cipher, path=path) + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Seal logs and key revisions before writing.""" + if self._is_log(path): + text = seal_log_text(text, self._cipher, path=path) + elif self._is_rev(path): + text = self.key.revision(text) + self._inner.put_file_text(path, text, message=message) + + def get_string_map(self, path: str) -> dict[str, str]: + """Bulk read; sealed entries come back opened, bad ones dropped.""" + reader = getattr(self._inner, "get_string_map", None) + if reader is None: + return {} + out: dict[str, str] = {} + for name, value in reader(path).items(): + full = f"{path}/{name}" + if not self._is_log(full): + out[name] = value + continue + try: + out[name] = open_log_text(value, self._cipher, path=full) + except E2eDecryptError: + _logger.warning("Undecryptable entry %s dropped", full) + return out + + def delete_file(self, path: str, *, message: str = "") -> None: + """Delegate.""" + self._inner.delete_file(path, message=message) + + def can_access_remote(self) -> bool: + """Delegate.""" + return self._inner.can_access_remote() + + +def _blobs(store: RemoteStore, root: str) -> dict[str, str]: + """Every text blob under ``root`` (flat leaves and ``/``).""" + skip = {default_revs_path(root), f"{root}/e2e", f"{root}/revs"} + out: dict[str, str] = {} + for name in store.list_directory(root): + full = f"{root}/{name}" + if full in skip: + continue + children = store.list_directory(full) + if not children: + text = store.get_file_text(full) + if text is not None: + out[name] = text + continue + for child in children: + text = store.get_file_text(f"{full}/{child}") + if text is not None: + out[f"{name}/{child}"] = text + return out + + +def copy_to_encrypted( + store: RemoteStore, key: E2eKey, old_prefix: str, layout: E2eLayout +) -> int: + """Merge every readable old log into its encrypted twin; return writes.""" + dest = EncryptingStore(store, key, layout) + copied = 0 + for rel, text in _blobs(store, old_prefix).items(): + try: + source = load_log(text) + except ValueError, KeyError, TypeError: + _logger.warning("Unreadable old log %s/%s not copied", old_prefix, rel) + continue + path = f"{layout.prefix}/{rel}" + existing = dest.get_file_text(path) + current = load_log(existing) if existing is not None else {} + merged = merge_logs(current, source) + if existing is not None and dump_log(merged) == dump_log(current): + continue + dest.put_file_text(path, dump_log(merged), message="e2e copy") + copied += 1 + return copied + + +_copied: dict[str, str] = {} + + +def reset_e2e_copy_memo() -> None: + """Forget which old prefixes were copied; for tests.""" + _copied.clear() + + +def _fingerprint(store: RemoteStore, old_prefix: str) -> str: + reader = getattr(store, "get_string_map", None) + if reader is None: + return "" + revs = reader(default_revs_path(old_prefix)) + return ";".join(f"{k}={revs[k]}" for k in sorted(revs)) + + +def open_encrypted_client( + store: RemoteStore, + old_prefix: str, + *, + passphrase: str, + params: Argon2Params | None = None, +) -> tuple[EncryptingStore, E2eLayout]: + """Unlock, copy the old prefix when it changed, and return the store. + + Raises: + E2eLockedError: No passphrase, a wrong one, or an unreadable header. + """ + layout = E2eLayout.of(old_prefix) + if not passphrase: + msg = "no crdt-sync passphrase on this device" + raise E2eLockedError(msg) + try: + key = unlock_e2e( + store, + path_prefix=layout.prefix, + passphrase=passphrase, + params=params, + e2e_path=layout.e2e_path, + ) + except (E2eWrongPassphraseError, E2eHeaderError) as exc: + raise E2eLockedError(str(exc) or type(exc).__name__) from exc + fingerprint = _fingerprint(store, old_prefix) + if _copied.get(old_prefix) != fingerprint: + copy_to_encrypted(store, key, old_prefix, layout) + _copied[old_prefix] = fingerprint + return EncryptingStore(store, key, layout), layout diff --git a/crdt-sync/crdt_sync/_pull.py b/crdt-sync/crdt_sync/_pull.py index 1b6c02a..ecdd100 100644 --- a/crdt-sync/crdt_sync/_pull.py +++ b/crdt-sync/crdt_sync/_pull.py @@ -13,6 +13,7 @@ from dataclasses import dataclass import logging from typing import TYPE_CHECKING +from crdt_sync._e2e_cipher import E2eDecryptError from crdt_sync._revisions import revision_of if TYPE_CHECKING: @@ -90,9 +91,16 @@ def _pull_remote_logs( # skipped next tick. seen_revs[other_device_id] = remote_rev continue - text = ctx.client.get_file_text( - f"{ctx.path_prefix}/{other_device_id}/{ctx.filename}" - ) + try: + text = ctx.client.get_file_text( + f"{ctx.path_prefix}/{other_device_id}/{ctx.filename}" + ) + except E2eDecryptError: + _logger.warning( + "Undecryptable log from device %r (wrong key or newer app)", + other_device_id, + ) + continue if text is None: continue try: diff --git a/crdt-sync/crdt_sync/tests/fixtures/e2e_dart_vectors.json b/crdt-sync/crdt_sync/tests/fixtures/e2e_dart_vectors.json new file mode 100644 index 0000000..87531bb --- /dev/null +++ b/crdt-sync/crdt_sync/tests/fixtures/e2e_dart_vectors.json @@ -0,0 +1,10 @@ +{ + "passphrase": "correct horse ✓", + "header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"HGW+FZJe6hd71He22BrlUA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"okKnATkNoZKvjSGfrgQGOjdRw5NORh2lC32zuBq7V6I=\"}", + "master_hex": "cad5afe8ddc6e21d2f39eb196de454db6b6d1f60629619ce8539f03a4bb0b086", + "revision_of_abc": "e7f845e1b0a7ea2b571fd3e3bdfb60641e650d63171bf73b6356c7909d2748c8", + "path": "vec/notes-e2e/phone/log.json", + "log": "{\"n1\":{\"id\":\"n1\",\"fields\":{\"title\":[\"buy milk\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null},\"n2\":{\"id\":\"n2\",\"fields\":{\"title\":[\"zażółć\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null}}", + "envelope": "{\"v\":1,\"r\":{\"n1\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"AR1zODx4TD67X3TGa0N7EV/V4FcZAaqc41jxZVtskFu3Pqb/pNlqm6db66rQUaMyNLtTPa4HMOJzZomBRyaSHjaPb4oNxgVPkMeIF5IbuI/GA30gR0sz/MSvYmfZ5wSnlBo1+fpH\"},\"n2\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"Aa0omlBGnmlsItzq7A21G0A1tqT8xMOJE0wgxcxJ1c3hVpRmNt4kq2IPNNGuW8v4A7fvs2FrnGX4A+NVtXPpu41ckKSShTqk8Ml6zTuxyrT85g50sgzbM7PKFczeEaS+ITWDj0N04u0=\"}}}", + "whole": "{\"v\":1,\"b\":\"AYdAwPehG84oHBRhRr5dP5VmleBGzQC1ryV5QHTCqBZE9ZsOZMMkJiP40+KzpnfIc2g=\"}" +} diff --git a/crdt-sync/crdt_sync/tests/test_e2e_cipher.py b/crdt-sync/crdt_sync/tests/test_e2e_cipher.py new file mode 100644 index 0000000..ec4806f --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_e2e_cipher.py @@ -0,0 +1,146 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""E2eCipher blob errors and the per-record envelope's edge shapes.""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +from crdt_sync import ( + E2eCipher, + E2eDecryptError, + is_e2e_envelope, + open_log_text, + seal_log_text, +) + +PATH = "app/devices-e2e/pc/log.json" +CLOCK = "2026-01-01T00:00:00.000Z-0000-pc" + + +def _cipher(*, accept_plaintext: bool = True) -> E2eCipher: + return E2eCipher(bytes(range(32)), accept_plaintext=accept_plaintext) + + +def _blob(raw: bytes) -> str: + return base64.b64encode(raw).decode() + + +def _log(**records: object) -> str: + return json.dumps(records) + + +def _record(rid: str = "a", **extra: object) -> dict[str, object]: + return { + "id": rid, + "fields": {"title": ["x", CLOCK]}, + "deleted": False, + "deleted_hlc": None, + **extra, + } + + +def test_key_must_be_32_bytes() -> None: + with pytest.raises(ValueError, match="32 bytes"): + E2eCipher(b"short") + + +def test_blob_round_trips() -> None: + cipher = _cipher() + assert cipher.open(cipher.seal("héllo", path=PATH), path=PATH) == "héllo" + + +def test_plaintext_is_read_only_while_accepted() -> None: + assert _cipher().open('{"a": 1}', path=PATH) == '{"a": 1}' + with pytest.raises(E2eDecryptError, match="after migration"): + _cipher(accept_plaintext=False).open("[1]", path=PATH) + + +@pytest.mark.parametrize( + ("stored", "message"), + [ + ("!!not base64!!", "neither JSON nor base64"), + ("", "empty blob"), + (_blob(b"\x02" + bytes(60)), "uses format 2"), + (_blob(b"\x01" + bytes(10)), "truncated"), + (_blob(b"\x01" + bytes(60)), "failed authentication"), + ], +) +def test_bad_blobs_are_refused(stored: str, message: str) -> None: + with pytest.raises(E2eDecryptError, match=message): + _cipher().open(stored, path=PATH) + + +def test_blob_is_bound_to_its_path() -> None: + sealed = _cipher().seal("secret", path=PATH) + with pytest.raises(E2eDecryptError, match="failed authentication"): + _cipher().open(sealed, path="other") + + +@pytest.mark.parametrize("text", ["not json", "[1]", '{"v": true}', '{"v": "1"}', "{}"]) +def test_non_envelopes_are_detected(text: str) -> None: + assert not is_e2e_envelope(text) + + +def test_envelope_is_detected() -> None: + assert is_e2e_envelope('{"v": 1}') + + +@pytest.mark.parametrize( + "text", + [ + "not json", + "[1]", + _log(a=[1]), + _log(a=_record("b")), + _log(a={"id": "a", "fields": []}), + ], +) +def test_non_log_text_is_sealed_whole(text: str) -> None: + sealed = seal_log_text(text, _cipher(), path=PATH) + assert "b" in json.loads(sealed) + assert open_log_text(sealed, _cipher(), path=PATH) == text + + +def test_record_without_fields_has_no_newest_clock() -> None: + empty = {"id": "a", "fields": {}, "deleted": True, "deleted_hlc": CLOCK} + sealed = seal_log_text(_log(a=empty), _cipher(), path=PATH) + assert json.loads(sealed)["r"]["a"]["h"] is None + opened = json.loads(open_log_text(sealed, _cipher(), path=PATH)) + assert opened == {"a": empty} + + +def test_plaintext_log_passes_or_fails_by_policy() -> None: + text = _log(a=_record()) + assert open_log_text(text, _cipher(), path=PATH) == text + with pytest.raises(E2eDecryptError, match="after migration"): + open_log_text(text, _cipher(accept_plaintext=False), path=PATH) + + +def test_unknown_envelope_version_is_refused() -> None: + with pytest.raises(E2eDecryptError, match="uses format 2"): + open_log_text('{"v": 2, "r": {}}', _cipher(), path=PATH) + + +def test_envelope_without_records_is_misshaped() -> None: + with pytest.raises(E2eDecryptError, match="mis-shaped"): + open_log_text('{"v": 1, "b": 5, "r": []}', _cipher(), path=PATH) + + +@pytest.mark.parametrize( + "record", + [[], {"p": "x", "d": 1}, {"d": False}, {"p": 3, "d": False}], +) +def test_misshaped_record_is_refused(record: object) -> None: + text = json.dumps({"v": 1, "r": {"a": record}}) + with pytest.raises(E2eDecryptError, match="record a"): + open_log_text(text, _cipher(), path=PATH) + + +def test_tampered_record_metadata_fails_authentication() -> None: + sealed = json.loads(seal_log_text(_log(a=_record()), _cipher(), path=PATH)) + sealed["r"]["a"]["d"] = True + with pytest.raises(E2eDecryptError, match="failed authentication"): + open_log_text(json.dumps(sealed), _cipher(), path=PATH) diff --git a/crdt-sync/crdt_sync/tests/test_e2e_interop.py b/crdt-sync/crdt_sync/tests/test_e2e_interop.py new file mode 100644 index 0000000..fe56309 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_e2e_interop.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Cross-language golden vectors: Python opens what Dart sealed.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from crdt_sync import ( + E2eDecryptError, + E2eKey, + E2eWrongPassphraseError, + open_log_text, + seal_log_text, + unlock_e2e, +) +from crdt_sync.tests.conftest import FakeStore as MemoryRemote + +WRONG = "nope" +VECTORS = json.loads( + (Path(__file__).parent / "fixtures" / "e2e_dart_vectors.json").read_text( + encoding="utf-8" + ) +) + + +def _key() -> E2eKey: + remote = MemoryRemote({"vec/e2e": VECTORS["header"]}) + return unlock_e2e( + remote, + path_prefix="vec/notes-e2e", + passphrase=str(VECTORS["passphrase"]), + e2e_path="vec/e2e", + ) + + +def test_argon2id_and_hkdf_match_dart() -> None: + assert _key().master.hex() == VECTORS["master_hex"] + + +def test_keyed_revision_matches_dart() -> None: + assert _key().revision("abc") == VECTORS["revision_of_abc"] + + +def test_opens_a_dart_envelope() -> None: + opened = open_log_text(VECTORS["envelope"], _key().cipher(), path=VECTORS["path"]) + assert json.loads(opened) == json.loads(VECTORS["log"]) + + +def test_opens_a_dart_whole_blob() -> None: + opened = open_log_text(VECTORS["whole"], _key().cipher(), path=VECTORS["path"]) + assert opened == "not a log" + + +def test_wrong_passphrase_is_refused() -> None: + remote = MemoryRemote({"vec/e2e": VECTORS["header"]}) + with pytest.raises(E2eWrongPassphraseError): + unlock_e2e(remote, path_prefix="x", passphrase=WRONG, e2e_path="vec/e2e") + + +def test_a_moved_envelope_is_refused() -> None: + with pytest.raises(E2eDecryptError): + open_log_text(VECTORS["envelope"], _key().cipher(), path="vec/other") + + +def test_python_round_trip() -> None: + cipher = _key().cipher() + sealed = seal_log_text(VECTORS["log"], cipher, path=VECTORS["path"]) + assert "buy milk" not in sealed + assert json.loads(open_log_text(sealed, cipher, path=VECTORS["path"])) == ( + json.loads(VECTORS["log"]) + ) diff --git a/crdt-sync/crdt_sync/tests/test_e2e_key.py b/crdt-sync/crdt_sync/tests/test_e2e_key.py new file mode 100644 index 0000000..a00c552 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_e2e_key.py @@ -0,0 +1,123 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Argon2id key, header parsing and unlock edge cases.""" + +from __future__ import annotations + +import json + +import pytest + +from crdt_sync import ( + Argon2Params, + E2eHeaderError, + E2eKey, + E2eWrongPassphraseError, + default_e2e_path, + unlock_e2e, +) +from crdt_sync._e2e_key import key_matches, read_header +from crdt_sync.tests.conftest import FakeStore + +CHEAP = Argon2Params(8, 1, 1) +HEADER = { + "v": 1, + "kdf": "argon2id", + "salt": "AAAAAAAAAAAAAAAAAAAAAA==", + "m": 8, + "t": 1, + "p": 1, + "check": "c", +} + + +def _unlock(remote: FakeStore, phrase: str = "pw") -> E2eKey: + return unlock_e2e( + remote, + path_prefix="app/devices-e2e", + passphrase=phrase, + params=CHEAP, + e2e_path="app/e2e", + ) + + +class RacedStore(FakeStore): + """Another device's header wins the first write.""" + + def __init__(self, winner: str) -> None: + super().__init__() + self._winner = winner + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + super().put_file_text(path, self._winner, message=message) + + +def test_default_path_is_a_sibling_or_nested() -> None: + assert default_e2e_path("app/devices-e2e") == "app/e2e" + assert default_e2e_path("flat") == "flat/e2e" + + +def test_master_key_must_be_32_bytes() -> None: + with pytest.raises(ValueError, match="32 bytes"): + E2eKey.from_bytes(b"short") + + +def test_unlock_refuses_an_empty_passphrase() -> None: + with pytest.raises(ValueError, match="empty"): + _unlock(FakeStore(), "") + + +def test_unlock_creates_then_reuses_the_header() -> None: + remote = FakeStore() + first = _unlock(remote) + assert _unlock(remote).master == first.master + assert key_matches(first, remote, "app/e2e") + assert not key_matches(_unlock(FakeStore(), "other"), remote, "app/e2e") + + +def test_missing_header_matches_nothing() -> None: + key = E2eKey.from_bytes(bytes(32)) + assert read_header(FakeStore(), "app/e2e") is None + assert not key_matches(key, FakeStore(), "app/e2e") + + +def test_wrong_passphrase_is_refused() -> None: + remote = FakeStore() + _unlock(remote) + with pytest.raises(E2eWrongPassphraseError): + _unlock(remote, "nope") + + +def test_losing_the_header_race_adopts_the_winner() -> None: + winner = FakeStore() + key = _unlock(winner) + raced = RacedStore(winner.files["app/e2e"]) + assert _unlock(raced).master == key.master + with pytest.raises(E2eWrongPassphraseError): + _unlock(RacedStore(winner.files["app/e2e"]), "nope") + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ("not json", "not JSON"), + ("[1]", "known version"), + (json.dumps({**HEADER, "v": 2}), "known version"), + (json.dumps({**HEADER, "kdf": "scrypt"}), "malformed"), + (json.dumps({k: v for k, v in HEADER.items() if k != "m"}), "malformed"), + (json.dumps({**HEADER, "m": "x"}), "malformed"), + (json.dumps({**HEADER, "salt": "A"}), "malformed"), + ], +) +def test_bad_headers_are_refused(text: str, message: str) -> None: + remote = FakeStore({"app/e2e": text}) + with pytest.raises(E2eHeaderError, match=message): + read_header(remote, "app/e2e") + with pytest.raises(E2eHeaderError, match=message): + _unlock(remote) + + +def test_a_good_header_parses() -> None: + header = read_header(FakeStore({"app/e2e": json.dumps(HEADER)}), "app/e2e") + assert header is not None + assert header.params == CHEAP + assert header.check == "c" diff --git a/crdt-sync/crdt_sync/tests/test_e2e_store.py b/crdt-sync/crdt_sync/tests/test_e2e_store.py new file mode 100644 index 0000000..ba96c3a --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_e2e_store.py @@ -0,0 +1,118 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""EncryptingStore, layout, one-time copy and open, mirroring the Dart side.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from crdt_sync import ( + Argon2Params, + E2eLayout, + E2eLockedError, + EncryptingStore, + LogCodec, + SyncTarget, + copy_to_encrypted, + dump_log, + load_log, + open_encrypted_client, + reset_e2e_copy_memo, + sync_log, + unlock_e2e, +) +from crdt_sync.tests.conftest import FakeStore + +if TYPE_CHECKING: + from collections.abc import Callable + + from crdt_sync import Hlc + +PHRASE = "pw" +WRONG = "bad" +CHEAP = Argon2Params(8, 1, 1) +OLD = "app/devices" + + +def _log_text(make_hlc: Callable[..., Hlc], **titles: str) -> str: + return json.dumps( + { + rid: { + "id": rid, + "fields": {"title": [t, make_hlc(1000).to_str()]}, + "deleted": False, + "deleted_hlc": None, + } + for rid, t in titles.items() + } + ) + + +@pytest.fixture(autouse=True) +def _memo() -> None: + reset_e2e_copy_memo() + + +def test_layouts() -> None: + assert E2eLayout.of("app/devices") == E2eLayout( + "app/devices-e2e", "app/revs-e2e", "app/e2e" + ) + assert E2eLayout.of("settings-sync") == E2eLayout( + "settings-sync-e2e", "settings-sync-e2e/revs", "settings-sync-e2e/e2e" + ) + + +def test_sync_through_store_is_ciphertext_and_merges( + make_hlc: Callable[..., Hlc], +) -> None: + remote = FakeStore() + layout = E2eLayout.of(OLD) + key = unlock_e2e( + remote, + path_prefix=layout.prefix, + passphrase=PHRASE, + params=CHEAP, + e2e_path=layout.e2e_path, + ) + store = EncryptingStore(remote, key, layout) + codec = LogCodec(encode=dump_log, decode=load_log, filename="log.json") + phone = load_log(_log_text(make_hlc, n1="buy milk")) + sync_log(SyncTarget(store, "phone", layout.prefix), phone, codec) + blob = remote.files[f"{layout.prefix}/phone/log.json"] + assert "buy milk" not in blob + assert '"title"' not in blob + merged = sync_log(SyncTarget(store, "pc", layout.prefix), {}, codec) + assert set(merged) == {"n1"} + + +def test_open_copies_old_prefix_once_and_locks_without_password( + make_hlc: Callable[..., Hlc], +) -> None: + remote = FakeStore({f"{OLD}/pc/log.json": _log_text(make_hlc, a="x")}) + with pytest.raises(E2eLockedError): + open_encrypted_client(remote, OLD, passphrase="", params=CHEAP) + store, layout = open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP) + text = store.get_file_text(f"{layout.prefix}/pc/log.json") + assert text is not None + assert set(load_log(text)) == {"a"} + assert "x" in remote.files[f"{OLD}/pc/log.json"] + with pytest.raises(E2eLockedError): + open_encrypted_client(remote, OLD, passphrase=WRONG, params=CHEAP) + + +def test_copy_is_idempotent(make_hlc: Callable[..., Hlc]) -> None: + remote = FakeStore({f"{OLD}/pc/log.json": _log_text(make_hlc, a="x")}) + layout = E2eLayout.of(OLD) + key = unlock_e2e( + remote, + path_prefix=layout.prefix, + passphrase=PHRASE, + params=CHEAP, + e2e_path=layout.e2e_path, + ) + assert copy_to_encrypted(remote, key, OLD, layout) == 1 + remote.writes.clear() + assert copy_to_encrypted(remote, key, OLD, layout) == 0 + assert remote.writes == [] diff --git a/crdt-sync/crdt_sync/tests/test_e2e_store_edges.py b/crdt-sync/crdt_sync/tests/test_e2e_store_edges.py new file mode 100644 index 0000000..7d804d9 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_e2e_store_edges.py @@ -0,0 +1,165 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""EncryptingStore passthroughs, bulk reads and copy/open edge cases.""" + +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING + +import pytest + +from crdt_sync import ( + Argon2Params, + E2eLayout, + E2eLockedError, + EncryptingStore, + copy_to_encrypted, + load_log, + open_encrypted_client, + reset_e2e_copy_memo, + unlock_e2e, +) +from crdt_sync._e2e_store import _blobs, _fingerprint +from crdt_sync.tests.conftest import FakeStore, FakeStoreWithoutBulkRead + +if TYPE_CHECKING: + from crdt_sync import E2eKey + +PHRASE = "pw" +CHEAP = Argon2Params(8, 1, 1) +OLD = "app/devices" +LAYOUT = E2eLayout.of(OLD) +CLOCK = "2026-01-01T00:00:00.000Z-0000-pc" + + +def _log(title: str = "x", rid: str = "a") -> str: + return json.dumps( + { + rid: { + "id": rid, + "fields": {"title": [title, CLOCK]}, + "deleted": False, + "deleted_hlc": None, + } + } + ) + + +def _key(remote: FakeStore) -> E2eKey: + return unlock_e2e( + remote, + path_prefix=LAYOUT.prefix, + passphrase=PHRASE, + params=CHEAP, + e2e_path=LAYOUT.e2e_path, + ) + + +class HidingStore(FakeStore): + """Lists every file but reads ``hidden`` paths as absent.""" + + def __init__(self, files: dict[str, str], hidden: set[str]) -> None: + super().__init__(files) + self._hidden = hidden + + def get_file_text(self, path: str) -> str | None: + if path in self._hidden: + return None + return super().get_file_text(path) + + +@pytest.fixture(autouse=True) +def _memo() -> None: + reset_e2e_copy_memo() + + +def test_passthroughs_and_keyed_revisions() -> None: + remote = FakeStore({"other/file": "raw"}) + store = EncryptingStore(remote, _key(remote), LAYOUT) + assert store.list_directory("other") == ["file"] + assert store.get_file_text("other/file") == "raw" + assert store.get_file_text(f"{LAYOUT.prefix}/pc/log.json") is None + assert store.can_access_remote() + store.put_file_text(f"{LAYOUT.revs_path}/pc", "text", message="m") + assert remote.files[f"{LAYOUT.revs_path}/pc"] == store.key.revision("text") + store.put_file_text("other/file", "plain", message="m") + assert remote.files["other/file"] == "plain" + store.delete_file("other/file") + assert "other/file" not in remote.files + + +def test_bulk_read_opens_sealed_and_drops_bad( + caplog: pytest.LogCaptureFixture, +) -> None: + remote = FakeStore() + store = EncryptingStore(remote, _key(remote), LAYOUT) + store.put_file_text(f"{LAYOUT.prefix}/pc", _log(), message="m") + remote.files[f"{LAYOUT.prefix}/bad"] = '{"v": 2, "r": {}}' + remote.files[f"{LAYOUT.revs_path}/pc"] = "rev" + with caplog.at_level(logging.WARNING): + out = store.get_string_map(LAYOUT.prefix) + assert set(out) == {"pc"} + assert set(load_log(out["pc"])) == {"a"} + assert "Undecryptable entry" in caplog.text + assert store.get_string_map(LAYOUT.revs_path) == {"pc": "rev"} + + +def test_bulk_read_is_empty_without_inner_support() -> None: + remote = FakeStoreWithoutBulkRead() + store = EncryptingStore(remote, _key(FakeStore()), LAYOUT) + assert store.get_string_map(LAYOUT.prefix) == {} + assert _fingerprint(remote, OLD) == "" + + +def test_blobs_cover_flat_nested_and_skipped_paths() -> None: + remote = HidingStore( + { + f"{OLD}/flat": "f", + f"{OLD}/gone": "g", + f"{OLD}/pc/log.json": "n", + f"{OLD}/pc/hidden.json": "h", + "app/revs/pc": "r", + f"{OLD}/e2e": "h", + f"{OLD}/revs": "r", + }, + {f"{OLD}/gone", f"{OLD}/pc/hidden.json"}, + ) + assert _blobs(remote, OLD) == {"flat": "f", "pc/log.json": "n"} + + +def test_copy_skips_unreadable_and_flat_logs( + caplog: pytest.LogCaptureFixture, +) -> None: + remote = FakeStore({f"{OLD}/flat": _log(), f"{OLD}/pc/log.json": "not json"}) + with caplog.at_level(logging.WARNING): + copied = copy_to_encrypted(remote, _key(remote), OLD, LAYOUT) + assert copied == 1 + assert "Unreadable old log" in caplog.text + assert f"{LAYOUT.prefix}/flat" in remote.files + assert f"{LAYOUT.prefix}/pc/log.json" not in remote.files + + +def test_changed_old_prefix_is_copied_again() -> None: + remote = FakeStore({f"{OLD}/pc/log.json": _log("one")}) + open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP) + remote.files[f"{OLD}/pc/log.json"] = _log("two", rid="b") + remote.files["app/revs/pc"] = "newer" + store, layout = open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP) + text = store.get_file_text(f"{layout.prefix}/pc/log.json") + assert text is not None + assert set(load_log(text)) == {"a", "b"} + + +def test_unchanged_prefix_is_not_copied_twice() -> None: + remote = FakeStore({f"{OLD}/pc/log.json": _log()}) + open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP) + remote.writes.clear() + open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP) + assert remote.writes == [] + + +def test_unreadable_header_locks_the_client() -> None: + remote = FakeStore({LAYOUT.e2e_path: "not json"}) + with pytest.raises(E2eLockedError, match="not JSON"): + open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP) diff --git a/crdt-sync/crdt_sync/tests/test_sync_e2e_skip.py b/crdt-sync/crdt_sync/tests/test_sync_e2e_skip.py new file mode 100644 index 0000000..9a07ec2 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_sync_e2e_skip.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""A peer whose log cannot be decrypted is skipped, not fatal.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +from crdt_sync import ( + E2eDecryptError, + GitHubSyncClient, + LogCodec, + SyncTarget, + dump_log, + load_log, + sync_log, +) + +if TYPE_CHECKING: + import pytest + + +def test_undecryptable_peer_is_skipped(caplog: pytest.LogCaptureFixture) -> None: + client = MagicMock(spec=GitHubSyncClient) + client.list_directory.return_value = ["phone"] + client.get_file_text.side_effect = E2eDecryptError("wrong key") + codec = LogCodec(encode=dump_log, decode=load_log, filename="log.json") + with caplog.at_level(logging.WARNING): + merged = sync_log(SyncTarget(client, "pc", "devices"), {}, codec) + assert merged == {} + assert "Undecryptable log from device 'phone'" in caplog.text diff --git a/crdt-sync/crdt_sync/tests/test_tool_e2e_vectors.py b/crdt-sync/crdt_sync/tests/test_tool_e2e_vectors.py new file mode 100644 index 0000000..500ceca --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_tool_e2e_vectors.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""tool/e2e_vectors writes vectors that Python itself can open.""" + +from __future__ import annotations + +import json +from pathlib import Path +import runpy + +import pytest + +from crdt_sync import open_log_text, unlock_e2e +from crdt_sync.tests.conftest import FakeStore +from tool import e2e_vectors + + +def test_vectors_open_with_the_stated_passphrase() -> None: + vec = e2e_vectors.vectors() + key = unlock_e2e( + FakeStore({"vec/e2e": str(vec["header"])}), + path_prefix="vec/notes-e2e", + passphrase=str(vec["passphrase"]), + e2e_path="vec/e2e", + ) + assert key.master.hex() == vec["master_hex"] + path = str(vec["path"]) + opened = open_log_text(str(vec["envelope"]), key.cipher(), path=path) + assert json.loads(opened) == json.loads(str(vec["log"])) + assert open_log_text(str(vec["whole"]), key.cipher(), path=path) == "not a log" + + +def test_main_prints_json(capsys: pytest.CaptureFixture[str]) -> None: + assert e2e_vectors.main() == 0 + assert "master_hex" in json.loads(capsys.readouterr().out) + + +def test_module_entry_point(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exc: + runpy.run_path(str(Path(e2e_vectors.__file__)), run_name="__main__") + assert exc.value.code == 0 + assert "envelope" in capsys.readouterr().out diff --git a/crdt-sync/pyproject.toml b/crdt-sync/pyproject.toml index 7fb4cd4..e668122 100644 --- a/crdt-sync/pyproject.toml +++ b/crdt-sync/pyproject.toml @@ -1,9 +1,13 @@ [project] name = "crdt-sync" -version = "0.9.2" +version = "0.10.0" description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps" requires-python = ">=3.14" -dependencies = ["requests==2.34.2"] +dependencies = [ + "argon2-cffi==25.1.0", + "pynacl==1.6.2", + "requests==2.34.2", +] # Ship only the library. `tool/` is a package too (it has an __init__.py so # its scripts import cleanly), and setuptools' flat-layout auto-discovery diff --git a/crdt-sync/requirements.txt b/crdt-sync/requirements.txt index cfc503a..8717d67 100644 --- a/crdt-sync/requirements.txt +++ b/crdt-sync/requirements.txt @@ -1,12 +1,14 @@ # Exact pins, enforced by the shared dependency-freshness gate: a `>=` # floor is satisfied by anything newer, so it tells a checker nothing # about whether this repo is actually current. +argon2-cffi==25.1.0 bandit[toml]==1.9.4 -mypy==2.4.0 +mypy==2.4.1 pre-commit==4.7.0 pylint==4.1.3 -pytest==9.1.1 +pynacl==1.6.2 +pytest==9.2.0 pytest-cov==7.1.0 requests==2.34.2 -ruff==0.17.2 -types-requests==2.33.0.20261006 +ruff==0.17.4 +types-requests==2.33.0.20261030 diff --git a/crdt-sync/tool/e2e_vectors.py b/crdt-sync/tool/e2e_vectors.py new file mode 100644 index 0000000..dcdcb39 --- /dev/null +++ b/crdt-sync/tool/e2e_vectors.py @@ -0,0 +1,64 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Write golden vectors sealed by Python for the Dart side to open. + +Run from ``crdt-sync``:: + + python -m tool.e2e_vectors > e2e_py_vectors.json + +then move the file to ``crdt_sync_dart/test/fixtures/``. +""" + +from __future__ import annotations + +import json +import sys + +from crdt_sync import Argon2Params, seal_log_text, unlock_e2e +from crdt_sync.tests.conftest import FakeStore + +# A public test-vector passphrase, not a secret. +PHRASE = "correct horse \u2713" + + +def vectors() -> dict[str, object]: + """Return a fresh vector set (random salt and nonces).""" + remote = FakeStore() + key = unlock_e2e( + remote, + path_prefix="vec/notes-e2e", + passphrase=PHRASE, + params=Argon2Params(64, 2, 1), + e2e_path="vec/e2e", + ) + path = "vec/notes-e2e/pc/log.json" + log = json.dumps( + { + "n9": { + "id": "n9", + "fields": {"title": ["zażółć", "2026-01-01T00:00:00.000Z-0000-pc"]}, + "deleted": False, + "deleted_hlc": None, + } + }, + ensure_ascii=False, + ) + cipher = key.cipher() + return { + "passphrase": PHRASE, + "header": remote.files["vec/e2e"], + "master_hex": key.master.hex(), + "path": path, + "log": log, + "envelope": seal_log_text(log, cipher, path=path), + "whole": seal_log_text("not a log", cipher, path=path), + } + + +def main() -> int: + """Print the vectors as JSON.""" + sys.stdout.write(json.dumps(vectors(), indent=2, ensure_ascii=False) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())