From edd0f7a7882cb1d3904932ba94faa282d08165df Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Thu, 12 Nov 2026 10:10:06 +0100 Subject: [PATCH] E2E copy never clobbers non-log blobs crdt_sync_dart 0.14.1, crdt-sync 0.11.0: the one-time copy skips empty sources and never overwrites a destination that is not a sync log (home-guard's pending upload). Python open_encrypted_client(state_dir=) caches the derived key (0600) and the copy memo across daemon runs. Todo: fda71ed1 Co-Authored-By: Claude Opus 5.5 --- crdt-sync/crdt_sync/_e2e_cache.py | 57 ++++++++++ crdt-sync/crdt_sync/_e2e_store.py | 91 +++++++++++++--- .../crdt_sync/tests/test_e2e_state_dir.py | 101 ++++++++++++++++++ .../crdt_sync/tests/test_e2e_store_edges.py | 54 +++++++++- crdt-sync/pyproject.toml | 2 +- crdt_sync_dart/lib/src/e2e_migration.dart | 3 + crdt_sync_dart/pubspec.yaml | 2 +- crdt_sync_dart/test/e2e_migration_test.dart | 18 ++++ 8 files changed, 308 insertions(+), 20 deletions(-) create mode 100644 crdt-sync/crdt_sync/_e2e_cache.py create mode 100644 crdt-sync/crdt_sync/tests/test_e2e_state_dir.py diff --git a/crdt-sync/crdt_sync/_e2e_cache.py b/crdt-sync/crdt_sync/_e2e_cache.py new file mode 100644 index 0000000..b1a57cb --- /dev/null +++ b/crdt-sync/crdt_sync/_e2e_cache.py @@ -0,0 +1,57 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""On-disk E2E state for short-lived daemons (one process per tick). + +The derived key is cached ``0600`` beside the app's other state, so Argon2id +runs once per machine rather than once a minute; it is no more sensitive +than ``~/.config/crdt-sync/password``, which it is derived from. The copy +memo records the old prefix's revisions last copied, so an unchanged old +prefix is not re-scanned every tick. +""" + +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +_MASTER = 32 + + +def _slug(old_prefix: str) -> str: + return hashlib.sha256(old_prefix.encode()).hexdigest()[:16] + + +def read_cached_key(state_dir: Path, old_prefix: str) -> bytes | None: + """Return the cached 32-byte master key, or None.""" + path = state_dir / f"e2e-{_slug(old_prefix)}.key" + if not path.exists(): + return None + try: + master = bytes.fromhex(path.read_text(encoding="utf-8").strip()) + except ValueError: + return None + return master if len(master) == _MASTER else None + + +def write_cached_key(state_dir: Path, old_prefix: str, master: bytes) -> None: + """Store the master key owner-only.""" + state_dir.mkdir(parents=True, exist_ok=True) + path = state_dir / f"e2e-{_slug(old_prefix)}.key" + path.touch(mode=0o600, exist_ok=True) + path.chmod(0o600) + path.write_text(master.hex(), encoding="utf-8") + + +def read_copy_memo(state_dir: Path, old_prefix: str) -> str | None: + """Return the old-prefix fingerprint last copied, or None.""" + path = state_dir / f"e2e-{_slug(old_prefix)}.copied" + return path.read_text(encoding="utf-8") if path.exists() else None + + +def write_copy_memo(state_dir: Path, old_prefix: str, fingerprint: str) -> None: + """Record that the old prefix was copied at ``fingerprint``.""" + state_dir.mkdir(parents=True, exist_ok=True) + path = state_dir / f"e2e-{_slug(old_prefix)}.copied" + path.write_text(fingerprint, encoding="utf-8") diff --git a/crdt-sync/crdt_sync/_e2e_store.py b/crdt-sync/crdt_sync/_e2e_store.py index 1f81ae1..d012294 100644 --- a/crdt-sync/crdt_sync/_e2e_store.py +++ b/crdt-sync/crdt_sync/_e2e_store.py @@ -12,12 +12,19 @@ from dataclasses import dataclass import logging from typing import TYPE_CHECKING +from crdt_sync._e2e_cache import ( + read_cached_key, + read_copy_memo, + write_cached_key, + write_copy_memo, +) from crdt_sync._e2e_cipher import E2eDecryptError, open_log_text, seal_log_text from crdt_sync._e2e_key import ( Argon2Params, E2eHeaderError, E2eKey, E2eWrongPassphraseError, + key_matches, unlock_e2e, ) from crdt_sync._log import merge_logs @@ -25,6 +32,8 @@ from crdt_sync._store import dump_log, load_log from crdt_sync._sync import default_revs_path if TYPE_CHECKING: + from pathlib import Path + from crdt_sync._remote import RemoteStore _logger = logging.getLogger(__name__) @@ -150,9 +159,16 @@ def copy_to_encrypted( except ValueError, KeyError, TypeError: _logger.warning("Unreadable old log %s/%s not copied", old_prefix, rel) continue + if not source: + continue path = f"{layout.prefix}/{rel}" existing = dest.get_file_text(path) - current = load_log(existing) if existing is not None else {} + try: + current = load_log(existing) if existing is not None else {} + except ValueError, KeyError, TypeError: + # Not a sync log (an app's own blob): never overwrite it. + _logger.warning("Non-log blob at %s left as is", path) + continue merged = merge_logs(current, source) if existing is not None and dump_log(merged) == dump_log(current): continue @@ -177,34 +193,75 @@ def _fingerprint(store: RemoteStore, old_prefix: str) -> str: return ";".join(f"{k}={revs[k]}" for k in sorted(revs)) -def open_encrypted_client( - store: RemoteStore, - old_prefix: str, - *, - passphrase: str, - params: Argon2Params | None = None, -) -> tuple[EncryptingStore, E2eLayout]: - """Unlock, copy the old prefix when it changed, and return the store. +@dataclass(frozen=True) +class _OpenRequest: + old_prefix: str + layout: E2eLayout + passphrase: str + params: Argon2Params | None + state_dir: Path | None - Raises: - E2eLockedError: No passphrase, a wrong one, or an unreadable header. - """ - layout = E2eLayout.of(old_prefix) - if not passphrase: + +def _unlock(store: RemoteStore, req: _OpenRequest) -> E2eKey: + old_prefix, layout, state_dir = req.old_prefix, req.layout, req.state_dir + if state_dir is not None: + cached = read_cached_key(state_dir, old_prefix) + if cached is not None: + key = E2eKey.from_bytes(cached) + if key_matches(key, store, layout.e2e_path): + return key + if not req.passphrase: msg = "no crdt-sync passphrase on this device" raise E2eLockedError(msg) try: key = unlock_e2e( store, path_prefix=layout.prefix, - passphrase=passphrase, - params=params, + passphrase=req.passphrase, + params=req.params, e2e_path=layout.e2e_path, ) except (E2eWrongPassphraseError, E2eHeaderError) as exc: raise E2eLockedError(str(exc) or type(exc).__name__) from exc + if state_dir is not None: + write_cached_key(state_dir, old_prefix, key.master) + return key + + +def open_encrypted_client( + store: RemoteStore, + old_prefix: str, + *, + passphrase: str, + params: Argon2Params | None = None, + state_dir: Path | None = None, +) -> tuple[EncryptingStore, E2eLayout]: + """Unlock, copy the old prefix when it changed, and return the store. + + Args: + store: The outermost remote (above any mirror). + old_prefix: The app's former plaintext prefix. + passphrase: The shared crdt-sync password. + params: Argon2id cost for a header created now; tests lower it. + state_dir: Where a short-lived daemon keeps its derived key (0600) + and copy memo between runs; None keeps both in memory only. + + Raises: + E2eLockedError: No passphrase, a wrong one, or an unreadable header. + """ + layout = E2eLayout.of(old_prefix) + key = _unlock( + store, _OpenRequest(old_prefix, layout, passphrase, params, state_dir) + ) fingerprint = _fingerprint(store, old_prefix) - if _copied.get(old_prefix) != fingerprint: + last = ( + read_copy_memo(state_dir, old_prefix) + if state_dir is not None + else _copied.get(old_prefix) + ) + if last != fingerprint: copy_to_encrypted(store, key, old_prefix, layout) _copied[old_prefix] = fingerprint + if state_dir is not None: + write_copy_memo(state_dir, old_prefix, fingerprint) return EncryptingStore(store, key, layout), layout diff --git a/crdt-sync/crdt_sync/tests/test_e2e_state_dir.py b/crdt-sync/crdt_sync/tests/test_e2e_state_dir.py new file mode 100644 index 0000000..27b1386 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_e2e_state_dir.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""A per-minute daemon must not pay Argon2id or a full copy every tick.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from crdt_sync import Argon2Params, E2eLockedError, open_encrypted_client +from crdt_sync import _e2e_key as key_mod +from crdt_sync import _e2e_store as store_mod +from crdt_sync.tests.conftest import FakeStore + +if TYPE_CHECKING: + from pathlib import Path + + from crdt_sync import E2eKey, RemoteStore + +CHEAP = Argon2Params(8, 1, 1) +PHRASE = "pw" +WRONG = PHRASE + "x" +LOG = json.dumps( + { + "a": { + "id": "a", + "fields": {"t": ["x", "2026-01-01T00:00:00.000Z-0000-a"]}, + "deleted": False, + "deleted_hlc": None, + } + } +) + + +def test_state_dir_caches_key_and_copy_across_processes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + remote = FakeStore({"app/devices/pc/log.json": LOG, "app/revs/pc": "r1"}) + derives: list[int] = [] + real = key_mod.unlock_e2e + + def counting( + remote: RemoteStore, + *, + path_prefix: str, + passphrase: str, + params: Argon2Params | None = None, + e2e_path: str | None = None, + ) -> E2eKey: + derives.append(1) + return real( + remote, + path_prefix=path_prefix, + passphrase=passphrase, + params=params, + e2e_path=e2e_path, + ) + + monkeypatch.setattr(store_mod, "unlock_e2e", counting) + for _ in range(3): + store_mod.reset_e2e_copy_memo() # a fresh process each tick + remote.writes.clear() + open_encrypted_client( + remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path + ) + assert derives == [1] + assert remote.writes == [] + key_files = list(tmp_path.glob("e2e-*.key")) + assert len(key_files) == 1 + assert key_files[0].stat().st_mode & 0o777 == 0o600 + + +def test_stale_cached_key_rederives(tmp_path: Path) -> None: + remote = FakeStore() + open_encrypted_client( + remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path + ) + for key_file in tmp_path.glob("e2e-*.key"): + key_file.write_text("00" * 32, encoding="utf-8") + store, _ = open_encrypted_client( + remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path + ) + assert next(tmp_path.glob("e2e-*.key")).read_text(encoding="utf-8") == ( + store.key.master.hex() + ) + + +def test_garbage_cached_key_is_ignored_and_wrong_passphrase_locks( + tmp_path: Path, +) -> None: + remote = FakeStore() + open_encrypted_client( + remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path + ) + for key_file in tmp_path.glob("e2e-*.key"): + key_file.write_text("not hex", encoding="utf-8") + with pytest.raises(E2eLockedError): + open_encrypted_client( + remote, "app/devices", passphrase=WRONG, params=CHEAP, state_dir=tmp_path + ) diff --git a/crdt-sync/crdt_sync/tests/test_e2e_store_edges.py b/crdt-sync/crdt_sync/tests/test_e2e_store_edges.py index 7d804d9..cc626bb 100644 --- a/crdt-sync/crdt_sync/tests/test_e2e_store_edges.py +++ b/crdt-sync/crdt_sync/tests/test_e2e_store_edges.py @@ -24,7 +24,9 @@ from crdt_sync._e2e_store import _blobs, _fingerprint from crdt_sync.tests.conftest import FakeStore, FakeStoreWithoutBulkRead if TYPE_CHECKING: - from crdt_sync import E2eKey + from collections.abc import Callable + + from crdt_sync import E2eKey, Hlc PHRASE = "pw" CHEAP = Argon2Params(8, 1, 1) @@ -163,3 +165,53 @@ def test_unreadable_header_locks_the_client() -> None: remote = FakeStore({LAYOUT.e2e_path: "not json"}) with pytest.raises(E2eLockedError, match="not JSON"): open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP) + + +def test_copy_never_overwrites_non_log_and_skips_empty() -> None: + layout = E2eLayout.of("app/devices") + remote = FakeStore( + { + "app/devices/pc/current.json": "{}", + "app/devices-e2e/pc/current.json": '{"photos":["a"]}', + "app/devices/tab/current.json": "{}", + } + ) + key = unlock_e2e( + remote, + path_prefix=layout.prefix, + passphrase="x" * 8, + params=Argon2Params(8, 1, 1), + e2e_path=layout.e2e_path, + ) + assert copy_to_encrypted(remote, key, "app/devices", layout) == 0 + assert remote.files["app/devices-e2e/pc/current.json"] == '{"photos":["a"]}' + assert "app/devices-e2e/tab/current.json" not in remote.files + + +def test_copy_leaves_a_non_log_destination_alone( + make_hlc: Callable[..., Hlc], +) -> None: + layout = E2eLayout.of("app/devices") + log = { + "n": { + "id": "n", + "fields": {"t": ["x", make_hlc(1).to_str()]}, + "deleted": False, + "deleted_hlc": None, + } + } + remote = FakeStore( + { + "app/devices/pc/current.json": json.dumps(log), + "app/devices-e2e/pc/current.json": '{"photos":["a"]}', + } + ) + key = unlock_e2e( + remote, + path_prefix=layout.prefix, + passphrase="x" * 8, + params=Argon2Params(8, 1, 1), + e2e_path=layout.e2e_path, + ) + assert copy_to_encrypted(remote, key, "app/devices", layout) == 0 + assert remote.files["app/devices-e2e/pc/current.json"] == '{"photos":["a"]}' diff --git a/crdt-sync/pyproject.toml b/crdt-sync/pyproject.toml index e668122..e71c14d 100644 --- a/crdt-sync/pyproject.toml +++ b/crdt-sync/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "crdt-sync" -version = "0.10.0" +version = "0.11.0" description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps" requires-python = ">=3.14" dependencies = [ diff --git a/crdt_sync_dart/lib/src/e2e_migration.dart b/crdt_sync_dart/lib/src/e2e_migration.dart index d7979a2..39afc46 100644 --- a/crdt_sync_dart/lib/src/e2e_migration.dart +++ b/crdt_sync_dart/lib/src/e2e_migration.dart @@ -155,9 +155,12 @@ Future copyToEncrypted({ skipped.add(rel); continue; } + if (source.isEmpty) continue; final path = '${layout.prefix}/$rel'; final existing = await dest.getFileText(path); final current = existing == null ? null : _parse(existing); + // Not a sync log (an app's own blob): never overwrite it. + if (existing != null && current == null) continue; final merged = mergeLogs(current ?? {}, source); if (current != null && _signature(merged) == _signature(current)) continue; await dest.putFileText(path, logToJson(merged), message: 'e2e copy'); diff --git a/crdt_sync_dart/pubspec.yaml b/crdt_sync_dart/pubspec.yaml index a8929ab..0abad8a 100644 --- a/crdt_sync_dart/pubspec.yaml +++ b/crdt_sync_dart/pubspec.yaml @@ -2,7 +2,7 @@ name: crdt_sync description: >- Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps. Pure Dart, no Flutter dependency. -version: 0.14.0 +version: 0.14.1 publish_to: 'none' environment: diff --git a/crdt_sync_dart/test/e2e_migration_test.dart b/crdt_sync_dart/test/e2e_migration_test.dart index 0193cf2..beaf31b 100644 --- a/crdt_sync_dart/test/e2e_migration_test.dart +++ b/crdt_sync_dart/test/e2e_migration_test.dart @@ -200,4 +200,22 @@ void main() { ); }); }); + + test('never overwrites a non-log blob and skips empty sources', () async { + final remote = MemoryRemote({ + 'app/notes/pc/current.json': '{}', + 'app/notes-e2e/pc/current.json': '{"photos":["a"]}', + 'app/notes/tab/current.json': '{}', + }); + final key = await _key(remote, layout); + final report = await copyToEncrypted( + store: remote, + key: key, + fromPrefix: _old, + layout: layout, + ); + expect(report.copied, 0); + expect(remote.files['app/notes-e2e/pc/current.json'], '{"photos":["a"]}'); + expect(remote.files.containsKey('app/notes-e2e/tab/current.json'), isFalse); + }); }