crdt_sync_dart 0.14.0: key cache, layout, interop

Slashless prefixes keep revs/header inside their node; copy walks
nested dirs (RTDB bulk reads are one level), reruns only when old revs
change; derived key cached; bulk reads decrypt; Python golden vectors.

Todo: fda71ed1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crdt_sync_dart/lib/src/e2e_migration.dart

@@ -13,6 +13,7 @@ import 'log.dart';
 import 'record.dart';
 import 'remote_store.dart';
 import 'store.dart';
+import 'sync.dart';
 
 /// Where an app's encrypted data lives, derived from its old plaintext prefix.
 class E2eLayout {
@@ -20,11 +21,17 @@ class E2eLayout {
 
   /// `<oldPrefix>-e2e`, with revisions and header apart from the old ones so
   /// old builds neither see nor clobber them.
+  ///
+  /// A slashless prefix (`settings-sync`) keeps revisions and header inside
+  /// its own node, as `defaultRevsPath` does, never at the database root.
   factory E2eLayout.of(String oldPrefix) {
-    final cut = oldPrefix.lastIndexOf('/');
-    final parent = cut < 0 ? '' : '${oldPrefix.substring(0, cut)}/';
     final prefix = '$oldPrefix-e2e';
-    return E2eLayout._(prefix, '${parent}revs-e2e', defaultE2ePath(prefix));
+    final cut = oldPrefix.lastIndexOf('/');
+    if (cut < 0) {
+      return E2eLayout._(prefix, '$prefix/revs', '$prefix/e2e');
+    }
+    final parent = oldPrefix.substring(0, cut);
+    return E2eLayout._(prefix, '$parent/revs-e2e', defaultE2ePath(prefix));
   }
 
   final String prefix;
@@ -42,26 +49,53 @@ class MigrationReport {
   final List<String> skipped;
 }
 
+/// Every text blob under [root], keyed by path relative to it: flat
+/// `<node>.json` leaves and nested `<device>/<file>` blobs. Real RTDB bulk
+/// reads return direct string children only, so dirs are walked.
 Future<Map<String, String>> _listBlobs(RemoteStore store, String root) async {
-  if (store is BulkMapReader) {
-    return (store as BulkMapReader).getStringMap(root);
-  }
   final out = <String, String>{};
-  for (final entry in await store.listDirectory(root)) {
-    final children = await store.listDirectory('$root/$entry');
+  if (store is BulkMapReader) {
+    out.addAll(await (store as BulkMapReader).getStringMap(root));
+  }
+  final revs = defaultRevsPath(root);
+  final header = defaultE2ePath(root);
+  for (final name in await store.listDirectory(root)) {
+    if (out.containsKey(name)) continue;
+    final full = '$root/$name';
+    if (full == header || full == revs) continue;
+    final children = await _safeList(store, full);
     if (children.isEmpty) {
-      final text = await store.getFileText('$root/$entry');
-      if (text != null) out[entry] = text;
+      final text = await _safeRead(store, full);
+      if (text != null) out[name] = text;
       continue;
     }
     for (final child in children) {
-      final text = await store.getFileText('$root/$entry/$child');
-      if (text != null) out['$entry/$child'] = text;
+      final text = await _safeRead(store, '$full/$child');
+      if (text != null) out['$name/$child'] = text;
     }
   }
+  out.removeWhere(
+    (rel, _) => '$root/$rel' == header || '$root/$rel'.startsWith('$revs/'),
+  );
   return out;
 }
 
+Future<List<String>> _safeList(RemoteStore store, String path) async {
+  try {
+    return await store.listDirectory(path);
+  } on Object {
+    return const [];
+  }
+}
+
+Future<String?> _safeRead(RemoteStore store, String path) async {
+  try {
+    return await store.getFileText(path);
+  } on Object {
+    return null;
+  }
+}
+
 EncryptingStore _dest(RemoteStore store, E2eKey key, E2eLayout layout) =>
     EncryptingStore(
       store,

crdt_sync_dart/lib/src/e2e_open.dart

@@ -4,9 +4,9 @@
 /// [readPassphrase] supplies the shared crdt-sync passphrase from the OS
 /// keystore, so the open is prompt-free. Fail closed: a missing or wrong
 /// passphrase, or an unreadable header, returns [E2eLocked] and the caller
-/// must not sync. On success the old plaintext prefix is copied over once
-/// per call (idempotent; picks up writes from builds not yet migrated) and
-/// is never deleted here.
+/// must not sync. On success the old plaintext prefix is copied over
+/// (idempotent; picks up writes from builds not yet migrated) and is never
+/// deleted here.
 library;
 
 import 'dart:developer';
@@ -14,51 +14,129 @@ import 'dart:developer';
 import 'e2e_key.dart';
 import 'e2e_migration.dart';
 import 'remote_store.dart';
+import 'sync.dart';
 
 /// Keystore entry name apps store the shared passphrase under.
 const String kE2ePassphraseStorageKey = 'crdt_sync_e2e_passphrase';
 
+/// Where a device keeps its derived 32-byte key, so Argon2id runs once per
+/// device rather than once per sync tick. Back it with the OS keystore.
+class E2eKeyCache {
+  /// Creates a cache from [read] and [write] closures.
+  const E2eKeyCache({required this.read, required this.write});
+
+  /// Returns the cached key bytes, or null.
+  final Future<List<int>?> Function() read;
+
+  /// Stores freshly derived key bytes.
+  final Future<void> Function(List<int> bytes) write;
+}
+
+/// Old-prefix revision fingerprints already copied, per old prefix.
+final Map<String, String> _copied = {};
+
+/// Forgets which old prefixes were copied; for tests.
+void resetE2eCopyMemo() => _copied.clear();
+
+Future<String> _oldFingerprint(RemoteStore store, String oldPrefix) async {
+  if (store is! BulkMapReader) return '';
+  try {
+    final revs = await (store as BulkMapReader).getStringMap(
+      defaultRevsPath(oldPrefix),
+    );
+    final keys = revs.keys.toList()..sort();
+    return [for (final k in keys) '$k=${revs[k]}'].join(';');
+  } on Object {
+    return '';
+  }
+}
+
+Future<E2eReady?> _tryCached(
+  RemoteStore store,
+  E2eLayout layout,
+  E2eKeyCache? cache,
+) async {
+  final bytes = await cache?.read();
+  if (bytes == null || bytes.length != 32) return null;
+  final state = await openE2e(
+    store: store,
+    pathPrefix: layout.prefix,
+    key: await E2eKey.fromBytes(bytes),
+    e2ePath: layout.e2ePath,
+    revsPath: layout.revsPath,
+  );
+  return state is E2eReady ? state : null;
+}
+
+/// Opens the encrypted store; see the library doc.
+///
+/// [keyCache] skips Argon2id when it holds the key for the current header.
+/// The one-time copy reruns only when the old prefix's published revisions
+/// change (or once per process where the backend cannot bulk-read them).
+/// [onDerive] is called each time Argon2id actually runs; for tests.
 Future<E2eState> openEncryptedStore({
   required RemoteStore store,
   required String oldPrefix,
   required Future<String?> Function() readPassphrase,
   Argon2Params params = Argon2Params.recommended,
+  E2eKeyCache? keyCache,
+  void Function()? onDerive,
 }) async {
-  final passphrase = await readPassphrase();
-  if (passphrase == null || passphrase.isEmpty) {
-    return E2eLocked(E2eLockReason.needsPassphrase);
-  }
   final layout = E2eLayout.of(oldPrefix);
-  final E2eKey key;
+  final E2eReady ready;
+  final E2eReady? cached;
   try {
-    key = await unlockE2e(
-      store: store,
-      pathPrefix: layout.prefix,
-      passphrase: passphrase,
-      params: params,
-      e2ePath: layout.e2ePath,
-    );
-  } on E2eWrongPassphraseError {
-    return E2eLocked(E2eLockReason.wrongKey);
+    cached = await _tryCached(store, layout, keyCache);
   } on E2eHeaderError {
     return E2eLocked(E2eLockReason.badHeader);
   }
-  try {
-    await copyToEncrypted(
+  if (cached != null) {
+    ready = cached;
+  } else {
+    final passphrase = await readPassphrase();
+    if (passphrase == null || passphrase.isEmpty) {
+      return E2eLocked(E2eLockReason.needsPassphrase);
+    }
+    final E2eKey key;
+    try {
+      onDerive?.call();
+      key = await unlockE2e(
+        store: store,
+        pathPrefix: layout.prefix,
+        passphrase: passphrase,
+        params: params,
+        e2ePath: layout.e2ePath,
+      );
+    } on E2eWrongPassphraseError {
+      return E2eLocked(E2eLockReason.wrongKey);
+    } on E2eHeaderError {
+      return E2eLocked(E2eLockReason.badHeader);
+    }
+    await keyCache?.write(key.bytes);
+    final state = await openE2e(
       store: store,
+      pathPrefix: layout.prefix,
       key: key,
-      fromPrefix: oldPrefix,
-      layout: layout,
+      e2ePath: layout.e2ePath,
+      revsPath: layout.revsPath,
     );
+    if (state is! E2eReady) return state;
+    ready = state;
+  }
+  try {
+    final print = await _oldFingerprint(store, oldPrefix);
+    if (_copied[oldPrefix] != print) {
+      await copyToEncrypted(
+        store: store,
+        key: ready.store.key,
+        fromPrefix: oldPrefix,
+        layout: layout,
+      );
+      _copied[oldPrefix] = print;
+    }
   } on Object catch (error, stackTrace) {
     // The copy is retried next open; sync itself must not stop for it.
     log('e2e copy failed', level: 900, error: error, stackTrace: stackTrace);
   }
-  return await openE2e(
-    store: store,
-    pathPrefix: layout.prefix,
-    key: key,
-    e2ePath: layout.e2ePath,
-    revsPath: layout.revsPath,
-  );
+  return ready;
 }

crdt_sync_dart/lib/src/encrypting_store.dart

@@ -26,6 +26,9 @@ class EncryptingStore implements RemoteStore, BulkMapReader {
 
   final RemoteStore _inner;
   final E2eKey _key;
+
+  /// The key this store seals with.
+  E2eKey get key => _key;
   final E2eCipher _cipher;
   final String _prefix;
   final String _revs;
@@ -60,7 +63,22 @@ class EncryptingStore implements RemoteStore, BulkMapReader {
   @override
   Future<Map<String, String>> getStringMap(String path) async {
     if (_inner case final BulkMapReader reader) {
-      return reader.getStringMap(path);
+      final map = await reader.getStringMap(path);
+      if (!_isLog('$path/x')) return map;
+      final out = <String, String>{};
+      for (final MapEntry(:key, :value) in map.entries) {
+        final full = '$path/$key';
+        if (!_isLog(full)) {
+          out[key] = value;
+          continue;
+        }
+        try {
+          out[key] = await openLogText(value, _cipher, path: full);
+        } on E2eDecryptError {
+          // Unreadable entries are dropped, as a corrupt value would be.
+        }
+      }
+      return out;
     }
     return const {};
   }

crdt_sync_dart/pubspec.yaml

@@ -2,7 +2,7 @@ name: crdt_sync
 description: >-
   Shared CRDT merge scheme + GitHub-Contents-API sync transport for
   cross-device apps. Pure Dart, no Flutter dependency.
-version: 0.13.1
+version: 0.14.0
 publish_to: 'none'
 
 environment:

crdt_sync_dart/test/e2e_fakes.dart

@@ -64,7 +64,8 @@ class MemoryRemote implements RemoteStore, BulkMapReader {
   @override
   Future<Map<String, String>> getStringMap(String path) async => {
     for (final entry in files.entries)
-      if (entry.key.startsWith('$path/'))
+      if (entry.key.startsWith('$path/') &&
+          !entry.key.substring(path.length + 1).contains('/'))
         entry.key.substring(path.length + 1): entry.value,
   };
 

crdt_sync_dart/test/e2e_interop_test.dart

@@ -0,0 +1,45 @@
+/// Cross-language golden vectors: Dart opens what the Python port sealed.
+library;
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:crdt_sync/crdt_sync.dart';
+import 'package:test/test.dart';
+
+import 'e2e_fakes.dart';
+
+void main() {
+  final vec =
+      jsonDecode(File('test/fixtures/e2e_py_vectors.json').readAsStringSync())
+          as Map<String, dynamic>;
+
+  Future<E2eKey> key() => unlockE2e(
+    store: MemoryRemote({'vec/e2e': vec['header'] as String}),
+    pathPrefix: 'vec/notes-e2e',
+    passphrase: vec['passphrase'] as String,
+    e2ePath: 'vec/e2e',
+  );
+
+  test('Argon2id and HKDF match Python', () async {
+    final hex = (await key()).bytes
+        .map((b) => b.toRadixString(16).padLeft(2, '0'))
+        .join();
+    expect(hex, vec['master_hex']);
+  });
+
+  test('opens a Python envelope and whole blob', () async {
+    final cipher = (await key()).cipher();
+    final path = vec['path'] as String;
+    final opened = await openLogText(
+      vec['envelope'] as String,
+      cipher,
+      path: path,
+    );
+    expect(jsonDecode(opened), jsonDecode(vec['log'] as String));
+    expect(
+      await openLogText(vec['whole'] as String, cipher, path: path),
+      'not a log',
+    );
+  });
+}

crdt_sync_dart/test/e2e_migration_test.dart

@@ -164,4 +164,40 @@ void main() {
       isFalse,
     );
   });
+
+  group('slashless prefix (settings-sync)', () {
+    final flat = E2eLayout.of('settings-sync');
+
+    test('revs and header stay inside the app node', () {
+      expect(flat.prefix, 'settings-sync-e2e');
+      expect(flat.revsPath, 'settings-sync-e2e/revs');
+      expect(flat.e2ePath, 'settings-sync-e2e/e2e');
+    });
+
+    test('old revs and header are not copied as logs', () async {
+      final remote = MemoryRemote({
+        'settings-sync/phone/log.json': encodeLog(logOf({'a': 'x'})),
+        'settings-sync/revs/phone': 'abc',
+        'settings-sync/e2e': '{"v":1}',
+      });
+      final key = await _key(remote, flat);
+      final report = await copyToEncrypted(
+        store: remote,
+        key: key,
+        fromPrefix: 'settings-sync',
+        layout: flat,
+      );
+      expect(report.copied, 1);
+      expect(report.skipped, isEmpty);
+      expect(
+        await verifyCopy(
+          store: remote,
+          key: key,
+          fromPrefix: 'settings-sync',
+          layout: flat,
+        ),
+        isTrue,
+      );
+    });
+  });
 }

crdt_sync_dart/test/e2e_open_cache_test.dart

@@ -0,0 +1,98 @@
+/// openEncryptedStore must not pay Argon2id or a full copy on every tick.
+library;
+
+import 'package:crdt_sync/crdt_sync.dart';
+import 'package:test/test.dart';
+
+import 'e2e_fakes.dart';
+
+const _old = 'app/notes';
+
+void main() {
+  setUp(resetE2eCopyMemo);
+
+  test('a cached key skips Argon2 and still unlocks', () async {
+    final remote = MemoryRemote();
+    List<int>? cached;
+    var derived = 0;
+    Future<E2eState> open() => openEncryptedStore(
+      store: remote,
+      oldPrefix: _old,
+      readPassphrase: () async => 'pw',
+      params: cheapArgon,
+      keyCache: E2eKeyCache(
+        read: () async => cached,
+        write: (bytes) async => cached = bytes,
+      ),
+      onDerive: () => derived++,
+    );
+    expect(await open(), isA<E2eReady>());
+    expect(cached, hasLength(32));
+    expect(await open(), isA<E2eReady>());
+    expect(derived, 1);
+  });
+
+  test('a stale cached key falls back to the passphrase', () async {
+    final remote = MemoryRemote();
+    List<int>? cached = List<int>.filled(32, 9);
+    final state = await openEncryptedStore(
+      store: remote,
+      oldPrefix: _old,
+      readPassphrase: () async => 'pw',
+      params: cheapArgon,
+      keyCache: E2eKeyCache(
+        read: () async => cached,
+        write: (bytes) async => cached = bytes,
+      ),
+    );
+    expect(state, isA<E2eReady>());
+    expect(cached, isNot(List<int>.filled(32, 9)));
+  });
+
+  test('the copy reruns only when old revisions change', () async {
+    final remote = MemoryRemote({
+      '$_old/pc/log.json': encodeLog(logOf({'a': 'x'})),
+      'app/revs/pc': 'r1',
+    });
+    Future<void> open() => openEncryptedStore(
+      store: remote,
+      oldPrefix: _old,
+      readPassphrase: () async => 'pw',
+      params: cheapArgon,
+    );
+    await open();
+    remote.reads.clear();
+    await open();
+    expect(remote.reads.where((p) => p.startsWith('app/notes-e2e/')), isEmpty);
+    remote.files['$_old/pc/log.json'] = encodeLog(logOf({'a': 'x', 'b': 'y'}));
+    remote.files['app/revs/pc'] = 'r2';
+    await open();
+    final state =
+        await openEncryptedStore(
+              store: remote,
+              oldPrefix: _old,
+              readPassphrase: () async => 'pw',
+              params: cheapArgon,
+            )
+            as E2eReady;
+    final text = await state.store.getFileText('app/notes-e2e/pc/log.json');
+    expect(decodeLog(text!).keys, unorderedEquals(['a', 'b']));
+  });
+
+  test('bulk reads of a data dir come back decrypted', () async {
+    final remote = MemoryRemote();
+    final state =
+        await openEncryptedStore(
+              store: remote,
+              oldPrefix: _old,
+              readPassphrase: () async => 'pw',
+              params: cheapArgon,
+            )
+            as E2eReady;
+    final log = encodeLog(logOf({'k': 'secret'}));
+    await state.store.putFileText('app/notes-e2e/k1', log, message: 'm');
+    expect(remote.files['app/notes-e2e/k1'], isNot(contains('secret')));
+    final map = await state.store.getStringMap('app/notes-e2e');
+    expect(map['k1'], log);
+  });
+}

crdt_sync_dart/test/fixtures/e2e_dart_vectors.json

@@ -0,0 +1,10 @@
+{
+  "passphrase": "correct horse ✓",
+  "header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"HGW+FZJe6hd71He22BrlUA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"okKnATkNoZKvjSGfrgQGOjdRw5NORh2lC32zuBq7V6I=\"}",
+  "master_hex": "cad5afe8ddc6e21d2f39eb196de454db6b6d1f60629619ce8539f03a4bb0b086",
+  "revision_of_abc": "e7f845e1b0a7ea2b571fd3e3bdfb60641e650d63171bf73b6356c7909d2748c8",
+  "path": "vec/notes-e2e/phone/log.json",
+  "log": "{\"n1\":{\"id\":\"n1\",\"fields\":{\"title\":[\"buy milk\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null},\"n2\":{\"id\":\"n2\",\"fields\":{\"title\":[\"zażółć\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null}}",
+  "envelope": "{\"v\":1,\"r\":{\"n1\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"AR1zODx4TD67X3TGa0N7EV/V4FcZAaqc41jxZVtskFu3Pqb/pNlqm6db66rQUaMyNLtTPa4HMOJzZomBRyaSHjaPb4oNxgVPkMeIF5IbuI/GA30gR0sz/MSvYmfZ5wSnlBo1+fpH\"},\"n2\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"Aa0omlBGnmlsItzq7A21G0A1tqT8xMOJE0wgxcxJ1c3hVpRmNt4kq2IPNNGuW8v4A7fvs2FrnGX4A+NVtXPpu41ckKSShTqk8Ml6zTuxyrT85g50sgzbM7PKFczeEaS+ITWDj0N04u0=\"}}}",
+  "whole": "{\"v\":1,\"b\":\"AYdAwPehG84oHBRhRr5dP5VmleBGzQC1ryV5QHTCqBZE9ZsOZMMkJiP40+KzpnfIc2g=\"}"
+}

crdt_sync_dart/test/fixtures/e2e_py_vectors.json

@@ -0,0 +1,9 @@
+{
+  "passphrase": "correct horse ✓",
+  "header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"fGy9SnWSTwxKbS/sAluVjA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"ekpcuZxZl+Ei6Y5yQp+cwdi4yR6ItJiYR7S3cd214ds=\"}",
+  "master_hex": "834f54fef4eb086e4201b918c938be179f91cf72e157e28d56dfe72bc46cbefd",
+  "path": "vec/notes-e2e/pc/log.json",
+  "log": "{\"n9\": {\"id\": \"n9\", \"fields\": {\"title\": [\"zażółć\", \"2026-01-01T00:00:00.000Z-0000-pc\"]}, \"deleted\": false, \"deleted_hlc\": null}}",
+  "envelope": "{\"v\":1,\"r\":{\"n9\":{\"d\":false,\"dh\":null,\"h\":\"2026-01-01T00:00:00.000Z-0000-pc\",\"p\":\"ATAIR4jQmlBjDF+/4+AYp3ATcGgjZQ60tQ05uvF1howYly4W4h3lLI04BssCRASargNV/+hDhqTl7n2cz4hh7Z5lmZYgElso/4k85hwO1SRuDPOYMKsLCpYgTy8zYzEiiRlv+g==\"}}}",
+  "whole": "{\"v\":1,\"b\":\"ARvUAj5nIfmJdsxdaiAl5uYkgry9VG5VQLQBi6+BNbyJ0t//UGjc1Ji2hmbBJl3v9kw=\"}"
+}

crdt_sync_dart/tool/e2e_vectors.dart

@@ -0,0 +1,41 @@
+/// Writes cross-language golden vectors for the Python port to open.
+///
+/// Run: `dart run tool/e2e_vectors.dart > test/fixtures/e2e_dart_vectors.json`
+library;
+
+import 'dart:convert';
+import 'dart:math';
+
+import 'package:crdt_sync/crdt_sync.dart';
+
+import '../test/e2e_fakes.dart';
+
+Future<void> main() async {
+  const params = Argon2Params(memoryKiB: 64, iterations: 2, parallelism: 1);
+  final remote = MemoryRemote();
+  final key = await unlockE2e(
+    store: remote,
+    pathPrefix: 'vec/notes-e2e',
+    passphrase: 'correct horse ✓',
+    params: params,
+    random: Random(7),
+    e2ePath: 'vec/e2e',
+  );
+  final cipher = key.cipher();
+  const path = 'vec/notes-e2e/phone/log.json';
+  final log = encodeLog(logOf({'n1': 'buy milk', 'n2': 'zażółć'}));
+  print(
+    const JsonEncoder.withIndent('  ').convert({
+      'passphrase': 'correct horse ✓',
+      'header': remote.files['vec/e2e'],
+      'master_hex': key.bytes
+          .map((b) => b.toRadixString(16).padLeft(2, '0'))
+          .join(),
+      'revision_of_abc': await key.revision('abc'),
+      'path': path,
+      'log': log,
+      'envelope': await sealLogText(log, cipher, path: path),
+      'whole': await sealLogText('not a log', cipher, path: path),
+    }),
+  );
+}