E2E copy never clobbers non-log blobs

crdt_sync_dart 0.14.1, crdt-sync 0.11.0: the one-time copy skips empty
sources and never overwrites a destination that is not a sync log
(home-guard's pending upload). Python open_encrypted_client(state_dir=)
caches the derived key (0600) and the copy memo across daemon runs.

Todo: fda71ed1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crdt-sync/crdt_sync/_e2e_cache.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2026 Krzysztof Rudnicki
+"""On-disk E2E state for short-lived daemons (one process per tick).
+
+The derived key is cached ``0600`` beside the app's other state, so Argon2id
+runs once per machine rather than once a minute; it is no more sensitive
+than ``~/.config/crdt-sync/password``, which it is derived from. The copy
+memo records the old prefix's revisions last copied, so an unchanged old
+prefix is not re-scanned every tick.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+_MASTER = 32
+
+
+def _slug(old_prefix: str) -> str:
+    return hashlib.sha256(old_prefix.encode()).hexdigest()[:16]
+
+
+def read_cached_key(state_dir: Path, old_prefix: str) -> bytes | None:
+    """Return the cached 32-byte master key, or None."""
+    path = state_dir / f"e2e-{_slug(old_prefix)}.key"
+    if not path.exists():
+        return None
+    try:
+        master = bytes.fromhex(path.read_text(encoding="utf-8").strip())
+    except ValueError:
+        return None
+    return master if len(master) == _MASTER else None
+
+
+def write_cached_key(state_dir: Path, old_prefix: str, master: bytes) -> None:
+    """Store the master key owner-only."""
+    state_dir.mkdir(parents=True, exist_ok=True)
+    path = state_dir / f"e2e-{_slug(old_prefix)}.key"
+    path.touch(mode=0o600, exist_ok=True)
+    path.chmod(0o600)
+    path.write_text(master.hex(), encoding="utf-8")
+
+
+def read_copy_memo(state_dir: Path, old_prefix: str) -> str | None:
+    """Return the old-prefix fingerprint last copied, or None."""
+    path = state_dir / f"e2e-{_slug(old_prefix)}.copied"
+    return path.read_text(encoding="utf-8") if path.exists() else None
+
+
+def write_copy_memo(state_dir: Path, old_prefix: str, fingerprint: str) -> None:
+    """Record that the old prefix was copied at ``fingerprint``."""
+    state_dir.mkdir(parents=True, exist_ok=True)
+    path = state_dir / f"e2e-{_slug(old_prefix)}.copied"
+    path.write_text(fingerprint, encoding="utf-8")

crdt-sync/crdt_sync/_e2e_store.py

@@ -12,12 +12,19 @@ from dataclasses import dataclass
 import logging
 from typing import TYPE_CHECKING
 
+from crdt_sync._e2e_cache import (
+    read_cached_key,
+    read_copy_memo,
+    write_cached_key,
+    write_copy_memo,
+)
 from crdt_sync._e2e_cipher import E2eDecryptError, open_log_text, seal_log_text
 from crdt_sync._e2e_key import (
     Argon2Params,
     E2eHeaderError,
     E2eKey,
     E2eWrongPassphraseError,
+    key_matches,
     unlock_e2e,
 )
 from crdt_sync._log import merge_logs
@@ -25,6 +32,8 @@ from crdt_sync._store import dump_log, load_log
 from crdt_sync._sync import default_revs_path
 
 if TYPE_CHECKING:
+    from pathlib import Path
+
     from crdt_sync._remote import RemoteStore
 
 _logger = logging.getLogger(__name__)
@@ -150,9 +159,16 @@ def copy_to_encrypted(
         except ValueError, KeyError, TypeError:
             _logger.warning("Unreadable old log %s/%s not copied", old_prefix, rel)
             continue
+        if not source:
+            continue
         path = f"{layout.prefix}/{rel}"
         existing = dest.get_file_text(path)
-        current = load_log(existing) if existing is not None else {}
+        try:
+            current = load_log(existing) if existing is not None else {}
+        except ValueError, KeyError, TypeError:
+            # Not a sync log (an app's own blob): never overwrite it.
+            _logger.warning("Non-log blob at %s left as is", path)
+            continue
         merged = merge_logs(current, source)
         if existing is not None and dump_log(merged) == dump_log(current):
             continue
@@ -177,34 +193,75 @@ def _fingerprint(store: RemoteStore, old_prefix: str) -> str:
     return ";".join(f"{k}={revs[k]}" for k in sorted(revs))
 
 
-def open_encrypted_client(
-    store: RemoteStore,
-    old_prefix: str,
-    *,
-    passphrase: str,
-    params: Argon2Params | None = None,
-) -> tuple[EncryptingStore, E2eLayout]:
-    """Unlock, copy the old prefix when it changed, and return the store.
+@dataclass(frozen=True)
+class _OpenRequest:
+    old_prefix: str
+    layout: E2eLayout
+    passphrase: str
+    params: Argon2Params | None
+    state_dir: Path | None
 
-    Raises:
-        E2eLockedError: No passphrase, a wrong one, or an unreadable header.
-    """
-    layout = E2eLayout.of(old_prefix)
-    if not passphrase:
+
+def _unlock(store: RemoteStore, req: _OpenRequest) -> E2eKey:
+    old_prefix, layout, state_dir = req.old_prefix, req.layout, req.state_dir
+    if state_dir is not None:
+        cached = read_cached_key(state_dir, old_prefix)
+        if cached is not None:
+            key = E2eKey.from_bytes(cached)
+            if key_matches(key, store, layout.e2e_path):
+                return key
+    if not req.passphrase:
         msg = "no crdt-sync passphrase on this device"
         raise E2eLockedError(msg)
     try:
         key = unlock_e2e(
             store,
             path_prefix=layout.prefix,
-            passphrase=passphrase,
-            params=params,
+            passphrase=req.passphrase,
+            params=req.params,
             e2e_path=layout.e2e_path,
         )
     except (E2eWrongPassphraseError, E2eHeaderError) as exc:
         raise E2eLockedError(str(exc) or type(exc).__name__) from exc
+    if state_dir is not None:
+        write_cached_key(state_dir, old_prefix, key.master)
+    return key
+
+
+def open_encrypted_client(
+    store: RemoteStore,
+    old_prefix: str,
+    *,
+    passphrase: str,
+    params: Argon2Params | None = None,
+    state_dir: Path | None = None,
+) -> tuple[EncryptingStore, E2eLayout]:
+    """Unlock, copy the old prefix when it changed, and return the store.
+
+    Args:
+        store: The outermost remote (above any mirror).
+        old_prefix: The app's former plaintext prefix.
+        passphrase: The shared crdt-sync password.
+        params: Argon2id cost for a header created now; tests lower it.
+        state_dir: Where a short-lived daemon keeps its derived key (0600)
+            and copy memo between runs; None keeps both in memory only.
+
+    Raises:
+        E2eLockedError: No passphrase, a wrong one, or an unreadable header.
+    """
+    layout = E2eLayout.of(old_prefix)
+    key = _unlock(
+        store, _OpenRequest(old_prefix, layout, passphrase, params, state_dir)
+    )
     fingerprint = _fingerprint(store, old_prefix)
-    if _copied.get(old_prefix) != fingerprint:
+    last = (
+        read_copy_memo(state_dir, old_prefix)
+        if state_dir is not None
+        else _copied.get(old_prefix)
+    )
+    if last != fingerprint:
         copy_to_encrypted(store, key, old_prefix, layout)
         _copied[old_prefix] = fingerprint
+        if state_dir is not None:
+            write_copy_memo(state_dir, old_prefix, fingerprint)
     return EncryptingStore(store, key, layout), layout

crdt-sync/crdt_sync/tests/test_e2e_state_dir.py

@@ -0,0 +1,101 @@
+# Copyright (c) 2026 Krzysztof Rudnicki
+"""A per-minute daemon must not pay Argon2id or a full copy every tick."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import pytest
+
+from crdt_sync import Argon2Params, E2eLockedError, open_encrypted_client
+from crdt_sync import _e2e_key as key_mod
+from crdt_sync import _e2e_store as store_mod
+from crdt_sync.tests.conftest import FakeStore
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from crdt_sync import E2eKey, RemoteStore
+
+CHEAP = Argon2Params(8, 1, 1)
+PHRASE = "pw"
+WRONG = PHRASE + "x"
+LOG = json.dumps(
+    {
+        "a": {
+            "id": "a",
+            "fields": {"t": ["x", "2026-01-01T00:00:00.000Z-0000-a"]},
+            "deleted": False,
+            "deleted_hlc": None,
+        }
+    }
+)
+
+
+def test_state_dir_caches_key_and_copy_across_processes(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    remote = FakeStore({"app/devices/pc/log.json": LOG, "app/revs/pc": "r1"})
+    derives: list[int] = []
+    real = key_mod.unlock_e2e
+
+    def counting(
+        remote: RemoteStore,
+        *,
+        path_prefix: str,
+        passphrase: str,
+        params: Argon2Params | None = None,
+        e2e_path: str | None = None,
+    ) -> E2eKey:
+        derives.append(1)
+        return real(
+            remote,
+            path_prefix=path_prefix,
+            passphrase=passphrase,
+            params=params,
+            e2e_path=e2e_path,
+        )
+
+    monkeypatch.setattr(store_mod, "unlock_e2e", counting)
+    for _ in range(3):
+        store_mod.reset_e2e_copy_memo()  # a fresh process each tick
+        remote.writes.clear()
+        open_encrypted_client(
+            remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
+        )
+    assert derives == [1]
+    assert remote.writes == []
+    key_files = list(tmp_path.glob("e2e-*.key"))
+    assert len(key_files) == 1
+    assert key_files[0].stat().st_mode & 0o777 == 0o600
+
+
+def test_stale_cached_key_rederives(tmp_path: Path) -> None:
+    remote = FakeStore()
+    open_encrypted_client(
+        remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
+    )
+    for key_file in tmp_path.glob("e2e-*.key"):
+        key_file.write_text("00" * 32, encoding="utf-8")
+    store, _ = open_encrypted_client(
+        remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
+    )
+    assert next(tmp_path.glob("e2e-*.key")).read_text(encoding="utf-8") == (
+        store.key.master.hex()
+    )
+
+
+def test_garbage_cached_key_is_ignored_and_wrong_passphrase_locks(
+    tmp_path: Path,
+) -> None:
+    remote = FakeStore()
+    open_encrypted_client(
+        remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
+    )
+    for key_file in tmp_path.glob("e2e-*.key"):
+        key_file.write_text("not hex", encoding="utf-8")
+    with pytest.raises(E2eLockedError):
+        open_encrypted_client(
+            remote, "app/devices", passphrase=WRONG, params=CHEAP, state_dir=tmp_path
+        )

crdt-sync/crdt_sync/tests/test_e2e_store_edges.py

@@ -24,7 +24,9 @@ from crdt_sync._e2e_store import _blobs, _fingerprint
 from crdt_sync.tests.conftest import FakeStore, FakeStoreWithoutBulkRead
 
 if TYPE_CHECKING:
-    from crdt_sync import E2eKey
+    from collections.abc import Callable
+
+    from crdt_sync import E2eKey, Hlc
 
 PHRASE = "pw"
 CHEAP = Argon2Params(8, 1, 1)
@@ -163,3 +165,53 @@ def test_unreadable_header_locks_the_client() -> None:
     remote = FakeStore({LAYOUT.e2e_path: "not json"})
     with pytest.raises(E2eLockedError, match="not JSON"):
         open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP)
+
+
+def test_copy_never_overwrites_non_log_and_skips_empty() -> None:
+    layout = E2eLayout.of("app/devices")
+    remote = FakeStore(
+        {
+            "app/devices/pc/current.json": "{}",
+            "app/devices-e2e/pc/current.json": '{"photos":["a"]}',
+            "app/devices/tab/current.json": "{}",
+        }
+    )
+    key = unlock_e2e(
+        remote,
+        path_prefix=layout.prefix,
+        passphrase="x" * 8,
+        params=Argon2Params(8, 1, 1),
+        e2e_path=layout.e2e_path,
+    )
+    assert copy_to_encrypted(remote, key, "app/devices", layout) == 0
+    assert remote.files["app/devices-e2e/pc/current.json"] == '{"photos":["a"]}'
+    assert "app/devices-e2e/tab/current.json" not in remote.files
+
+
+def test_copy_leaves_a_non_log_destination_alone(
+    make_hlc: Callable[..., Hlc],
+) -> None:
+    layout = E2eLayout.of("app/devices")
+    log = {
+        "n": {
+            "id": "n",
+            "fields": {"t": ["x", make_hlc(1).to_str()]},
+            "deleted": False,
+            "deleted_hlc": None,
+        }
+    }
+    remote = FakeStore(
+        {
+            "app/devices/pc/current.json": json.dumps(log),
+            "app/devices-e2e/pc/current.json": '{"photos":["a"]}',
+        }
+    )
+    key = unlock_e2e(
+        remote,
+        path_prefix=layout.prefix,
+        passphrase="x" * 8,
+        params=Argon2Params(8, 1, 1),
+        e2e_path=layout.e2e_path,
+    )
+    assert copy_to_encrypted(remote, key, "app/devices", layout) == 0
+    assert remote.files["app/devices-e2e/pc/current.json"] == '{"photos":["a"]}'

crdt-sync/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "crdt-sync"
-version = "0.10.0"
+version = "0.11.0"
 description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps"
 requires-python = ">=3.14"
 dependencies = [

crdt_sync_dart/lib/src/e2e_migration.dart

@@ -155,9 +155,12 @@ Future<MigrationReport> copyToEncrypted({
       skipped.add(rel);
       continue;
     }
+    if (source.isEmpty) continue;
     final path = '${layout.prefix}/$rel';
     final existing = await dest.getFileText(path);
     final current = existing == null ? null : _parse(existing);
+    // Not a sync log (an app's own blob): never overwrite it.
+    if (existing != null && current == null) continue;
     final merged = mergeLogs(current ?? <String, Record>{}, source);
     if (current != null && _signature(merged) == _signature(current)) continue;
     await dest.putFileText(path, logToJson(merged), message: 'e2e copy');

crdt_sync_dart/pubspec.yaml

@@ -2,7 +2,7 @@ name: crdt_sync
 description: >-
   Shared CRDT merge scheme + GitHub-Contents-API sync transport for
   cross-device apps. Pure Dart, no Flutter dependency.
-version: 0.14.0
+version: 0.14.1
 publish_to: 'none'
 
 environment:

crdt_sync_dart/test/e2e_migration_test.dart

@@ -200,4 +200,22 @@ void main() {
       );
     });
   });
+
+  test('never overwrites a non-log blob and skips empty sources', () async {
+    final remote = MemoryRemote({
+      'app/notes/pc/current.json': '{}',
+      'app/notes-e2e/pc/current.json': '{"photos":["a"]}',
+      'app/notes/tab/current.json': '{}',
+    });
+    final key = await _key(remote, layout);
+    final report = await copyToEncrypted(
+      store: remote,
+      key: key,
+      fromPrefix: _old,
+      layout: layout,
+    );
+    expect(report.copied, 0);
+    expect(remote.files['app/notes-e2e/pc/current.json'], '{"photos":["a"]}');
+    expect(remote.files.containsKey('app/notes-e2e/tab/current.json'), isFalse);
+  });
 }