diff --git a/crdt_sync_dart/lib/src/e2e_migration.dart b/crdt_sync_dart/lib/src/e2e_migration.dart index c3e31dc..d7979a2 100644 --- a/crdt_sync_dart/lib/src/e2e_migration.dart +++ b/crdt_sync_dart/lib/src/e2e_migration.dart @@ -13,6 +13,7 @@ import 'log.dart'; import 'record.dart'; import 'remote_store.dart'; import 'store.dart'; +import 'sync.dart'; /// Where an app's encrypted data lives, derived from its old plaintext prefix. class E2eLayout { @@ -20,11 +21,17 @@ class E2eLayout { /// `-e2e`, with revisions and header apart from the old ones so /// old builds neither see nor clobber them. + /// + /// A slashless prefix (`settings-sync`) keeps revisions and header inside + /// its own node, as `defaultRevsPath` does, never at the database root. factory E2eLayout.of(String oldPrefix) { - final cut = oldPrefix.lastIndexOf('/'); - final parent = cut < 0 ? '' : '${oldPrefix.substring(0, cut)}/'; final prefix = '$oldPrefix-e2e'; - return E2eLayout._(prefix, '${parent}revs-e2e', defaultE2ePath(prefix)); + final cut = oldPrefix.lastIndexOf('/'); + if (cut < 0) { + return E2eLayout._(prefix, '$prefix/revs', '$prefix/e2e'); + } + final parent = oldPrefix.substring(0, cut); + return E2eLayout._(prefix, '$parent/revs-e2e', defaultE2ePath(prefix)); } final String prefix; @@ -42,26 +49,53 @@ class MigrationReport { final List skipped; } +/// Every text blob under [root], keyed by path relative to it: flat +/// `.json` leaves and nested `/` blobs. Real RTDB bulk +/// reads return direct string children only, so dirs are walked. Future> _listBlobs(RemoteStore store, String root) async { - if (store is BulkMapReader) { - return (store as BulkMapReader).getStringMap(root); - } final out = {}; - for (final entry in await store.listDirectory(root)) { - final children = await store.listDirectory('$root/$entry'); + if (store is BulkMapReader) { + out.addAll(await (store as BulkMapReader).getStringMap(root)); + } + final revs = defaultRevsPath(root); + final header = defaultE2ePath(root); + for (final name in await store.listDirectory(root)) { + if (out.containsKey(name)) continue; + final full = '$root/$name'; + if (full == header || full == revs) continue; + final children = await _safeList(store, full); if (children.isEmpty) { - final text = await store.getFileText('$root/$entry'); - if (text != null) out[entry] = text; + final text = await _safeRead(store, full); + if (text != null) out[name] = text; continue; } for (final child in children) { - final text = await store.getFileText('$root/$entry/$child'); - if (text != null) out['$entry/$child'] = text; + final text = await _safeRead(store, '$full/$child'); + if (text != null) out['$name/$child'] = text; } } + out.removeWhere( + (rel, _) => '$root/$rel' == header || '$root/$rel'.startsWith('$revs/'), + ); return out; } +Future> _safeList(RemoteStore store, String path) async { + try { + return await store.listDirectory(path); + } on Object { + return const []; + } +} + +Future _safeRead(RemoteStore store, String path) async { + try { + return await store.getFileText(path); + } on Object { + return null; + } +} + EncryptingStore _dest(RemoteStore store, E2eKey key, E2eLayout layout) => EncryptingStore( store, diff --git a/crdt_sync_dart/lib/src/e2e_open.dart b/crdt_sync_dart/lib/src/e2e_open.dart index 992cb7d..ac5b1a4 100644 --- a/crdt_sync_dart/lib/src/e2e_open.dart +++ b/crdt_sync_dart/lib/src/e2e_open.dart @@ -4,9 +4,9 @@ /// [readPassphrase] supplies the shared crdt-sync passphrase from the OS /// keystore, so the open is prompt-free. Fail closed: a missing or wrong /// passphrase, or an unreadable header, returns [E2eLocked] and the caller -/// must not sync. On success the old plaintext prefix is copied over once -/// per call (idempotent; picks up writes from builds not yet migrated) and -/// is never deleted here. +/// must not sync. On success the old plaintext prefix is copied over +/// (idempotent; picks up writes from builds not yet migrated) and is never +/// deleted here. library; import 'dart:developer'; @@ -14,51 +14,129 @@ import 'dart:developer'; import 'e2e_key.dart'; import 'e2e_migration.dart'; import 'remote_store.dart'; +import 'sync.dart'; /// Keystore entry name apps store the shared passphrase under. const String kE2ePassphraseStorageKey = 'crdt_sync_e2e_passphrase'; +/// Where a device keeps its derived 32-byte key, so Argon2id runs once per +/// device rather than once per sync tick. Back it with the OS keystore. +class E2eKeyCache { + /// Creates a cache from [read] and [write] closures. + const E2eKeyCache({required this.read, required this.write}); + + /// Returns the cached key bytes, or null. + final Future?> Function() read; + + /// Stores freshly derived key bytes. + final Future Function(List bytes) write; +} + +/// Old-prefix revision fingerprints already copied, per old prefix. +final Map _copied = {}; + +/// Forgets which old prefixes were copied; for tests. +void resetE2eCopyMemo() => _copied.clear(); + +Future _oldFingerprint(RemoteStore store, String oldPrefix) async { + if (store is! BulkMapReader) return ''; + try { + final revs = await (store as BulkMapReader).getStringMap( + defaultRevsPath(oldPrefix), + ); + final keys = revs.keys.toList()..sort(); + return [for (final k in keys) '$k=${revs[k]}'].join(';'); + } on Object { + return ''; + } +} + +Future _tryCached( + RemoteStore store, + E2eLayout layout, + E2eKeyCache? cache, +) async { + final bytes = await cache?.read(); + if (bytes == null || bytes.length != 32) return null; + final state = await openE2e( + store: store, + pathPrefix: layout.prefix, + key: await E2eKey.fromBytes(bytes), + e2ePath: layout.e2ePath, + revsPath: layout.revsPath, + ); + return state is E2eReady ? state : null; +} + +/// Opens the encrypted store; see the library doc. +/// +/// [keyCache] skips Argon2id when it holds the key for the current header. +/// The one-time copy reruns only when the old prefix's published revisions +/// change (or once per process where the backend cannot bulk-read them). +/// [onDerive] is called each time Argon2id actually runs; for tests. Future openEncryptedStore({ required RemoteStore store, required String oldPrefix, required Future Function() readPassphrase, Argon2Params params = Argon2Params.recommended, + E2eKeyCache? keyCache, + void Function()? onDerive, }) async { - final passphrase = await readPassphrase(); - if (passphrase == null || passphrase.isEmpty) { - return E2eLocked(E2eLockReason.needsPassphrase); - } final layout = E2eLayout.of(oldPrefix); - final E2eKey key; + final E2eReady ready; + final E2eReady? cached; try { - key = await unlockE2e( - store: store, - pathPrefix: layout.prefix, - passphrase: passphrase, - params: params, - e2ePath: layout.e2ePath, - ); - } on E2eWrongPassphraseError { - return E2eLocked(E2eLockReason.wrongKey); + cached = await _tryCached(store, layout, keyCache); } on E2eHeaderError { return E2eLocked(E2eLockReason.badHeader); } - try { - await copyToEncrypted( + if (cached != null) { + ready = cached; + } else { + final passphrase = await readPassphrase(); + if (passphrase == null || passphrase.isEmpty) { + return E2eLocked(E2eLockReason.needsPassphrase); + } + final E2eKey key; + try { + onDerive?.call(); + key = await unlockE2e( + store: store, + pathPrefix: layout.prefix, + passphrase: passphrase, + params: params, + e2ePath: layout.e2ePath, + ); + } on E2eWrongPassphraseError { + return E2eLocked(E2eLockReason.wrongKey); + } on E2eHeaderError { + return E2eLocked(E2eLockReason.badHeader); + } + await keyCache?.write(key.bytes); + final state = await openE2e( store: store, + pathPrefix: layout.prefix, key: key, - fromPrefix: oldPrefix, - layout: layout, + e2ePath: layout.e2ePath, + revsPath: layout.revsPath, ); + if (state is! E2eReady) return state; + ready = state; + } + try { + final print = await _oldFingerprint(store, oldPrefix); + if (_copied[oldPrefix] != print) { + await copyToEncrypted( + store: store, + key: ready.store.key, + fromPrefix: oldPrefix, + layout: layout, + ); + _copied[oldPrefix] = print; + } } on Object catch (error, stackTrace) { // The copy is retried next open; sync itself must not stop for it. log('e2e copy failed', level: 900, error: error, stackTrace: stackTrace); } - return await openE2e( - store: store, - pathPrefix: layout.prefix, - key: key, - e2ePath: layout.e2ePath, - revsPath: layout.revsPath, - ); + return ready; } diff --git a/crdt_sync_dart/lib/src/encrypting_store.dart b/crdt_sync_dart/lib/src/encrypting_store.dart index 7230709..0a635ca 100644 --- a/crdt_sync_dart/lib/src/encrypting_store.dart +++ b/crdt_sync_dart/lib/src/encrypting_store.dart @@ -26,6 +26,9 @@ class EncryptingStore implements RemoteStore, BulkMapReader { final RemoteStore _inner; final E2eKey _key; + + /// The key this store seals with. + E2eKey get key => _key; final E2eCipher _cipher; final String _prefix; final String _revs; @@ -60,7 +63,22 @@ class EncryptingStore implements RemoteStore, BulkMapReader { @override Future> getStringMap(String path) async { if (_inner case final BulkMapReader reader) { - return reader.getStringMap(path); + final map = await reader.getStringMap(path); + if (!_isLog('$path/x')) return map; + final out = {}; + for (final MapEntry(:key, :value) in map.entries) { + final full = '$path/$key'; + if (!_isLog(full)) { + out[key] = value; + continue; + } + try { + out[key] = await openLogText(value, _cipher, path: full); + } on E2eDecryptError { + // Unreadable entries are dropped, as a corrupt value would be. + } + } + return out; } return const {}; } diff --git a/crdt_sync_dart/pubspec.yaml b/crdt_sync_dart/pubspec.yaml index f31a5f5..a8929ab 100644 --- a/crdt_sync_dart/pubspec.yaml +++ b/crdt_sync_dart/pubspec.yaml @@ -2,7 +2,7 @@ name: crdt_sync description: >- Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps. Pure Dart, no Flutter dependency. -version: 0.13.1 +version: 0.14.0 publish_to: 'none' environment: diff --git a/crdt_sync_dart/test/e2e_fakes.dart b/crdt_sync_dart/test/e2e_fakes.dart index 3ff8dce..10a0770 100644 --- a/crdt_sync_dart/test/e2e_fakes.dart +++ b/crdt_sync_dart/test/e2e_fakes.dart @@ -64,7 +64,8 @@ class MemoryRemote implements RemoteStore, BulkMapReader { @override Future> getStringMap(String path) async => { for (final entry in files.entries) - if (entry.key.startsWith('$path/')) + if (entry.key.startsWith('$path/') && + !entry.key.substring(path.length + 1).contains('/')) entry.key.substring(path.length + 1): entry.value, }; diff --git a/crdt_sync_dart/test/e2e_interop_test.dart b/crdt_sync_dart/test/e2e_interop_test.dart new file mode 100644 index 0000000..274a964 --- /dev/null +++ b/crdt_sync_dart/test/e2e_interop_test.dart @@ -0,0 +1,45 @@ +/// Cross-language golden vectors: Dart opens what the Python port sealed. +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:test/test.dart'; + +import 'e2e_fakes.dart'; + +void main() { + final vec = + jsonDecode(File('test/fixtures/e2e_py_vectors.json').readAsStringSync()) + as Map; + + Future key() => unlockE2e( + store: MemoryRemote({'vec/e2e': vec['header'] as String}), + pathPrefix: 'vec/notes-e2e', + passphrase: vec['passphrase'] as String, + e2ePath: 'vec/e2e', + ); + + test('Argon2id and HKDF match Python', () async { + final hex = (await key()).bytes + .map((b) => b.toRadixString(16).padLeft(2, '0')) + .join(); + expect(hex, vec['master_hex']); + }); + + test('opens a Python envelope and whole blob', () async { + final cipher = (await key()).cipher(); + final path = vec['path'] as String; + final opened = await openLogText( + vec['envelope'] as String, + cipher, + path: path, + ); + expect(jsonDecode(opened), jsonDecode(vec['log'] as String)); + expect( + await openLogText(vec['whole'] as String, cipher, path: path), + 'not a log', + ); + }); +} diff --git a/crdt_sync_dart/test/e2e_migration_test.dart b/crdt_sync_dart/test/e2e_migration_test.dart index 4c40726..0193cf2 100644 --- a/crdt_sync_dart/test/e2e_migration_test.dart +++ b/crdt_sync_dart/test/e2e_migration_test.dart @@ -164,4 +164,40 @@ void main() { isFalse, ); }); + + group('slashless prefix (settings-sync)', () { + final flat = E2eLayout.of('settings-sync'); + + test('revs and header stay inside the app node', () { + expect(flat.prefix, 'settings-sync-e2e'); + expect(flat.revsPath, 'settings-sync-e2e/revs'); + expect(flat.e2ePath, 'settings-sync-e2e/e2e'); + }); + + test('old revs and header are not copied as logs', () async { + final remote = MemoryRemote({ + 'settings-sync/phone/log.json': encodeLog(logOf({'a': 'x'})), + 'settings-sync/revs/phone': 'abc', + 'settings-sync/e2e': '{"v":1}', + }); + final key = await _key(remote, flat); + final report = await copyToEncrypted( + store: remote, + key: key, + fromPrefix: 'settings-sync', + layout: flat, + ); + expect(report.copied, 1); + expect(report.skipped, isEmpty); + expect( + await verifyCopy( + store: remote, + key: key, + fromPrefix: 'settings-sync', + layout: flat, + ), + isTrue, + ); + }); + }); } diff --git a/crdt_sync_dart/test/e2e_open_cache_test.dart b/crdt_sync_dart/test/e2e_open_cache_test.dart new file mode 100644 index 0000000..c593f85 --- /dev/null +++ b/crdt_sync_dart/test/e2e_open_cache_test.dart @@ -0,0 +1,98 @@ +/// openEncryptedStore must not pay Argon2id or a full copy on every tick. +library; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:test/test.dart'; + +import 'e2e_fakes.dart'; + +const _old = 'app/notes'; + +void main() { + setUp(resetE2eCopyMemo); + + test('a cached key skips Argon2 and still unlocks', () async { + final remote = MemoryRemote(); + List? cached; + var derived = 0; + Future open() => openEncryptedStore( + store: remote, + oldPrefix: _old, + readPassphrase: () async => 'pw', + params: cheapArgon, + keyCache: E2eKeyCache( + read: () async => cached, + write: (bytes) async => cached = bytes, + ), + onDerive: () => derived++, + ); + expect(await open(), isA()); + expect(cached, hasLength(32)); + expect(await open(), isA()); + expect(derived, 1); + }); + + test('a stale cached key falls back to the passphrase', () async { + final remote = MemoryRemote(); + List? cached = List.filled(32, 9); + final state = await openEncryptedStore( + store: remote, + oldPrefix: _old, + readPassphrase: () async => 'pw', + params: cheapArgon, + keyCache: E2eKeyCache( + read: () async => cached, + write: (bytes) async => cached = bytes, + ), + ); + expect(state, isA()); + expect(cached, isNot(List.filled(32, 9))); + }); + + test('the copy reruns only when old revisions change', () async { + final remote = MemoryRemote({ + '$_old/pc/log.json': encodeLog(logOf({'a': 'x'})), + 'app/revs/pc': 'r1', + }); + Future open() => openEncryptedStore( + store: remote, + oldPrefix: _old, + readPassphrase: () async => 'pw', + params: cheapArgon, + ); + await open(); + remote.reads.clear(); + await open(); + expect(remote.reads.where((p) => p.startsWith('app/notes-e2e/')), isEmpty); + remote.files['$_old/pc/log.json'] = encodeLog(logOf({'a': 'x', 'b': 'y'})); + remote.files['app/revs/pc'] = 'r2'; + await open(); + final state = + await openEncryptedStore( + store: remote, + oldPrefix: _old, + readPassphrase: () async => 'pw', + params: cheapArgon, + ) + as E2eReady; + final text = await state.store.getFileText('app/notes-e2e/pc/log.json'); + expect(decodeLog(text!).keys, unorderedEquals(['a', 'b'])); + }); + + test('bulk reads of a data dir come back decrypted', () async { + final remote = MemoryRemote(); + final state = + await openEncryptedStore( + store: remote, + oldPrefix: _old, + readPassphrase: () async => 'pw', + params: cheapArgon, + ) + as E2eReady; + final log = encodeLog(logOf({'k': 'secret'})); + await state.store.putFileText('app/notes-e2e/k1', log, message: 'm'); + expect(remote.files['app/notes-e2e/k1'], isNot(contains('secret'))); + final map = await state.store.getStringMap('app/notes-e2e'); + expect(map['k1'], log); + }); +} diff --git a/crdt_sync_dart/test/fixtures/e2e_dart_vectors.json b/crdt_sync_dart/test/fixtures/e2e_dart_vectors.json new file mode 100644 index 0000000..87531bb --- /dev/null +++ b/crdt_sync_dart/test/fixtures/e2e_dart_vectors.json @@ -0,0 +1,10 @@ +{ + "passphrase": "correct horse ✓", + "header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"HGW+FZJe6hd71He22BrlUA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"okKnATkNoZKvjSGfrgQGOjdRw5NORh2lC32zuBq7V6I=\"}", + "master_hex": "cad5afe8ddc6e21d2f39eb196de454db6b6d1f60629619ce8539f03a4bb0b086", + "revision_of_abc": "e7f845e1b0a7ea2b571fd3e3bdfb60641e650d63171bf73b6356c7909d2748c8", + "path": "vec/notes-e2e/phone/log.json", + "log": "{\"n1\":{\"id\":\"n1\",\"fields\":{\"title\":[\"buy milk\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null},\"n2\":{\"id\":\"n2\",\"fields\":{\"title\":[\"zażółć\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null}}", + "envelope": "{\"v\":1,\"r\":{\"n1\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"AR1zODx4TD67X3TGa0N7EV/V4FcZAaqc41jxZVtskFu3Pqb/pNlqm6db66rQUaMyNLtTPa4HMOJzZomBRyaSHjaPb4oNxgVPkMeIF5IbuI/GA30gR0sz/MSvYmfZ5wSnlBo1+fpH\"},\"n2\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"Aa0omlBGnmlsItzq7A21G0A1tqT8xMOJE0wgxcxJ1c3hVpRmNt4kq2IPNNGuW8v4A7fvs2FrnGX4A+NVtXPpu41ckKSShTqk8Ml6zTuxyrT85g50sgzbM7PKFczeEaS+ITWDj0N04u0=\"}}}", + "whole": "{\"v\":1,\"b\":\"AYdAwPehG84oHBRhRr5dP5VmleBGzQC1ryV5QHTCqBZE9ZsOZMMkJiP40+KzpnfIc2g=\"}" +} diff --git a/crdt_sync_dart/test/fixtures/e2e_py_vectors.json b/crdt_sync_dart/test/fixtures/e2e_py_vectors.json new file mode 100644 index 0000000..ad1723f --- /dev/null +++ b/crdt_sync_dart/test/fixtures/e2e_py_vectors.json @@ -0,0 +1,9 @@ +{ + "passphrase": "correct horse ✓", + "header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"fGy9SnWSTwxKbS/sAluVjA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"ekpcuZxZl+Ei6Y5yQp+cwdi4yR6ItJiYR7S3cd214ds=\"}", + "master_hex": "834f54fef4eb086e4201b918c938be179f91cf72e157e28d56dfe72bc46cbefd", + "path": "vec/notes-e2e/pc/log.json", + "log": "{\"n9\": {\"id\": \"n9\", \"fields\": {\"title\": [\"zażółć\", \"2026-01-01T00:00:00.000Z-0000-pc\"]}, \"deleted\": false, \"deleted_hlc\": null}}", + "envelope": "{\"v\":1,\"r\":{\"n9\":{\"d\":false,\"dh\":null,\"h\":\"2026-01-01T00:00:00.000Z-0000-pc\",\"p\":\"ATAIR4jQmlBjDF+/4+AYp3ATcGgjZQ60tQ05uvF1howYly4W4h3lLI04BssCRASargNV/+hDhqTl7n2cz4hh7Z5lmZYgElso/4k85hwO1SRuDPOYMKsLCpYgTy8zYzEiiRlv+g==\"}}}", + "whole": "{\"v\":1,\"b\":\"ARvUAj5nIfmJdsxdaiAl5uYkgry9VG5VQLQBi6+BNbyJ0t//UGjc1Ji2hmbBJl3v9kw=\"}" +} diff --git a/crdt_sync_dart/tool/e2e_vectors.dart b/crdt_sync_dart/tool/e2e_vectors.dart new file mode 100644 index 0000000..fdf8e57 --- /dev/null +++ b/crdt_sync_dart/tool/e2e_vectors.dart @@ -0,0 +1,41 @@ +/// Writes cross-language golden vectors for the Python port to open. +/// +/// Run: `dart run tool/e2e_vectors.dart > test/fixtures/e2e_dart_vectors.json` +library; + +import 'dart:convert'; +import 'dart:math'; + +import 'package:crdt_sync/crdt_sync.dart'; + +import '../test/e2e_fakes.dart'; + +Future main() async { + const params = Argon2Params(memoryKiB: 64, iterations: 2, parallelism: 1); + final remote = MemoryRemote(); + final key = await unlockE2e( + store: remote, + pathPrefix: 'vec/notes-e2e', + passphrase: 'correct horse ✓', + params: params, + random: Random(7), + e2ePath: 'vec/e2e', + ); + final cipher = key.cipher(); + const path = 'vec/notes-e2e/phone/log.json'; + final log = encodeLog(logOf({'n1': 'buy milk', 'n2': 'zażółć'})); + print( + const JsonEncoder.withIndent(' ').convert({ + 'passphrase': 'correct horse ✓', + 'header': remote.files['vec/e2e'], + 'master_hex': key.bytes + .map((b) => b.toRadixString(16).padLeft(2, '0')) + .join(), + 'revision_of_abc': await key.revision('abc'), + 'path': path, + 'log': log, + 'envelope': await sealLogText(log, cipher, path: path), + 'whole': await sealLogText('not a log', cipher, path: path), + }), + ); +}