diff --git a/lib/sync/off_thread.dart b/lib/sync/off_thread.dart new file mode 100644 index 0000000..889778b --- /dev/null +++ b/lib/sync/off_thread.dart @@ -0,0 +1,7 @@ +/// Runs CPU-heavy, pure work away from the UI thread where the platform can. +/// +/// Conditional export because `dart:isolate` does not run in a browser: the +/// web desktop build runs the work inline, everything else on a fresh isolate. +library; + +export 'off_thread_io.dart' if (dart.library.js_interop) 'off_thread_web.dart'; diff --git a/lib/sync/off_thread_io.dart b/lib/sync/off_thread_io.dart new file mode 100644 index 0000000..d731c20 --- /dev/null +++ b/lib/sync/off_thread_io.dart @@ -0,0 +1,7 @@ +import 'dart:isolate'; + +/// Runs [work] on a short-lived isolate and returns its result. +/// +/// The result comes back through `Isolate.exit`, so a large parsed log is +/// handed over rather than copied. +Future offThread(R Function() work) => Isolate.run(work); diff --git a/lib/sync/off_thread_web.dart b/lib/sync/off_thread_web.dart new file mode 100644 index 0000000..9314fa3 --- /dev/null +++ b/lib/sync/off_thread_web.dart @@ -0,0 +1,2 @@ +/// Runs [work] inline: a browser has no isolates for it. +Future offThread(R Function() work) async => work(); diff --git a/lib/sync/peer_pull.dart b/lib/sync/peer_pull.dart new file mode 100644 index 0000000..79e9b05 --- /dev/null +++ b/lib/sync/peer_pull.dart @@ -0,0 +1,78 @@ +/// Pulls the peers whose logs changed, all at once and off the UI thread. +/// +/// Split out of `sync_service.dart` for file size. A peer's log is about +/// 2 MB; pulling eight of them one after another, each parsed on the UI +/// isolate, kept the phone's sync spinner up for 3.9 s (todo:0c575b891b). +library; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:todo/sync/off_thread.dart'; + +/// One peer file's pull result: its decoded [log] and revision, or a null +/// [log] when the file vanished, failed decryption or did not parse. +typedef PulledPeer = ({String name, String peer, Log? log, String? rev}); + +/// Fetches [names] (file names under [dir]) concurrently. +/// +/// [knownRevs] holds each peer's published revision; a peer without one gets +/// its revision computed from the downloaded text. Any failure other than an +/// [E2eDecryptError] fails the pull, exactly as a sequential loop would. +Future> pullPeers( + RemoteStore remote, + String dir, + List names, + Map knownRevs, +) => Future.wait([ + for (final name in names) _pull(remote, dir, name, knownRevs[peerOf(name)]), +], eagerError: true); + +/// Peer id of a `.json` file name. +String peerOf(String name) => + name.endsWith('.json') ? name.substring(0, name.length - 5) : name; + +Future _pull( + RemoteStore remote, + String dir, + String name, + String? knownRev, +) async { + final peer = peerOf(name); + final String? text; + try { + text = await remote.getFileText('$dir/$name'); + } on E2eDecryptError { + // Wrong key, tampered blob or a newer format: skipped and reported, + // never merged and never fatal. + return (name: name, peer: peer, log: null, rev: null); + } + if (text == null) return (name: name, peer: peer, log: null, rev: null); + final (log, rev) = await _decodeOffThread(text, knownRev); + return (name: name, peer: peer, log: log, rev: rev); +} + +/// Below this many characters a log parses faster than an isolate starts. +const int _offThreadMinLength = 64 * 1024; + +/// Kept separate so the isolate closure captures only [text] and [knownRev]. +Future<(Log?, String)> _decodeOffThread(String text, String? knownRev) async { + (Log?, String) decode() => (decodeLog(text), knownRev ?? revisionOf(text)); + return text.length < _offThreadMinLength ? decode() : await offThread(decode); +} + +/// Parses a remote note log, returning `null` for a corrupt or wrong-shape +/// payload so one bad device file never aborts the whole sync. +Log? decodeLog(String text) { + try { + return logFromJson(text); + } on FormatException { + return null; + } + // crdt_sync's logFromJson throws a raw TypeError (not a custom + // exception) for a wrong-shape JSON payload from an untrusted peer + // file; must be caught too so one bad device file never aborts the + // whole sync (see doc comment above). + // ignore: avoid_catching_errors + on TypeError { + return null; + } +} diff --git a/lib/sync/run_sync.dart b/lib/sync/run_sync.dart index 8d89934..2a25daf 100644 --- a/lib/sync/run_sync.dart +++ b/lib/sync/run_sync.dart @@ -88,9 +88,13 @@ Future runSync( stateStore: stateStore ?? await openSyncStateStore(), // coverage:ignore-end ); - final backend = firebase == null - ? github - : MirrorStore(primary: firebase, mirror: github); + // No GitHub token means nothing to mirror: an unconfigured client still + // spends two round trips per call on 404s. + final backend = switch (firebase) { + null => github, + _ when !settings.isConfigured => firebase, + _ => MirrorStore(primary: firebase, mirror: github), + }; final e2e = await openEncryptedStore( store: backend, oldPrefix: service.notesDir, @@ -106,15 +110,26 @@ Future runSync( ); } final layout = E2eLayout.of(service.notesDir); - final result = await SyncService( + // Settings and analytics touch other Firebase paths than the notes, so + // they ride alongside the note sync instead of after it. + final reconciled = appSettings?.reconcileWithRemote(firebase); + final flushed = analytics?.flush(firebase); + final notes = SyncService( notesDir: layout.prefix, revsPath: layout.revsPath, stateStore: service.stateStore, ).sync(repository, e2e.store); - await analytics?.flush(firebase); + final done = await Future.wait([ + notes, + ?reconciled, + ?flushed, + // Not eager: the clients are closed below, so nothing may still be in + // flight when this returns or throws. + ]); + final result = done.first! as SyncResult; return SyncRunResult( syncResult: result.withFirebaseConnected(value: firebase != null), - appSettings: await appSettings?.reconcileWithRemote(firebase), + appSettings: await reconciled, ); } finally { // Close both: unlike the long-lived desktop callers, each screen builds diff --git a/lib/sync/sync_service.dart b/lib/sync/sync_service.dart index f51dd55..6cb87ad 100644 --- a/lib/sync/sync_service.dart +++ b/lib/sync/sync_service.dart @@ -1,6 +1,7 @@ import 'package:crdt_sync/crdt_sync.dart'; import 'package:todo/data/note_repository.dart'; +import 'package:todo/sync/peer_pull.dart'; /// Outcome of a sync run, for surfacing in the UI. class SyncResult { @@ -113,53 +114,61 @@ class SyncService { final ownFileName = '$nodeId.json'; final state = await stateStore?.load() ?? const SyncState(); - final remoteRevs = stateStore == null - ? const {} - : await _revisions(remote); + // Independent reads, so one round trip instead of two. eagerError keeps + // the first failure's own type, which callers catch. + final reads = await Future.wait([ + if (stateStore == null) + Future.value(const {}) + else + _revisions(remote), + remote.listDirectory(notesDir), + ], eagerError: true); + final remoteRevs = reads[0] as Map; + final names = reads[1] as List; final seenRevs = {}; - // 1. Pull: list all device log files, merge everyone else's. - final names = await remote.listDirectory(notesDir); - var merged = 0; + // 1. Pull: every other device's log that changed since we merged it. var skippedUnchanged = 0; - final skipped = []; + final changed = []; for (final name in names) { if (name == ownFileName) continue; // our own file; skip it. - final peer = name.endsWith('.json') - ? name.substring(0, name.length - 5) - : name; + final peer = peerOf(name); final remoteRev = remoteRevs[peer]; if (remoteRev != null && remoteRev == state.peerRevs[peer]) { // Unchanged since we last merged it, and that merge is already in the - // local log — so the ~150 KB download is pure waste. Carry the - // revision forward so it stays skipped next time. + // local log -- so the download is pure waste. Carry the revision + // forward so it stays skipped next time. seenRevs[peer] = remoteRev; skippedUnchanged++; continue; } - final String? text; - try { - text = await remote.getFileText('$notesDir/$name'); - } on E2eDecryptError { - // Wrong key, tampered blob or a newer format: skipped and reported, - // never merged and never fatal. - skipped.add(name); + changed.add(name); + } + var merged = 0; + final skipped = []; + Log? incoming; + for (final pulled in await pullPeers( + remote, + notesDir, + changed, + remoteRevs, + )) { + final log = pulled.log; + if (log == null) { + // A vanished, undecryptable or undecodable peer file must not abort + // the sync, but it must not vanish silently either -- that's how a + // device drops out of sync unnoticed. Reported, and deliberately not + // recorded as seen: a corrupt push must be retried next run. + skipped.add(pulled.name); continue; } - final remoteLog = text == null ? null : _decodeLog(text); - if (remoteLog == null) { - // A vanished or undecodable peer file must not abort the sync, but - // it must not vanish silently either — that's how a device drops - // out of sync unnoticed. Report it in the result instead. - // Deliberately not recorded as seen: a corrupt push must be retried - // next run, not remembered as merged. - skipped.add(name); - continue; - } - await repo.importLog(remoteLog); - seenRevs[peer] = remoteRev ?? revisionOf(text!); + // Peers merge with each other first (commutative, idempotent), so the + // local store is rewritten once rather than once per peer. + incoming = incoming == null ? log : mergeLogs(incoming, log); + seenRevs[pulled.peer] = pulled.rev!; merged++; } + if (incoming != null) await repo.importLog(incoming); // 2. Push: upload our own (now-merged) log under our node id. // putFileText resolves our file's current sha itself. @@ -206,22 +215,4 @@ class SyncService { if (remote is! BulkMapReader) return const {}; return await (remote as BulkMapReader).getStringMap(revsDir); } - - /// Parses a remote note log, returning `null` for a corrupt or wrong-shape - /// payload so one bad device file never aborts the whole sync. - Log? _decodeLog(String text) { - try { - return logFromJson(text); - } on FormatException { - return null; - } - // crdt_sync's logFromJson throws a raw TypeError (not a custom - // exception) for a wrong-shape JSON payload from an untrusted peer - // file; must be caught too so one bad device file never aborts the - // whole sync (see doc comment above). - // ignore: avoid_catching_errors - on TypeError { - return null; - } - } } diff --git a/test/run_sync_latency_test.dart b/test/run_sync_latency_test.dart new file mode 100644 index 0000000..9086034 --- /dev/null +++ b/test/run_sync_latency_test.dart @@ -0,0 +1,105 @@ +/// The sync spinner must clear in under a second on a phone link. +/// +/// Every request to either backend waits [_rtt], a mobile round trip, so the +/// elapsed time measures how many round trips sit on the critical path. The +/// board reported a 3.9 s `sync.run` on the Pixel (todo:0c575b891b). +@TestOn('vm') +library; + +import 'dart:convert'; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart' as http_testing; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:todo/data/app_settings.dart'; +import 'package:todo/data/note_repository.dart'; +import 'package:todo/sync/run_sync.dart'; +import 'package:todo/sync/sync_settings.dart'; + +import 'fake_secure_storage.dart'; + +const _rtt = Duration(milliseconds: 150); + +/// Firebase RTDB stand-in: a path-keyed JSON store behind a slow link. +class _SlowFirebase { + final data = {}; + final requests = []; + + http.Client client() => http_testing.MockClient((request) async { + requests.add('${request.method} ${request.url.path}'); + await Future.delayed(_rtt); + final key = request.url.path; + if (request.method == 'PUT') { + data[key] = jsonDecode(request.body); + return http.Response(request.body, 200); + } + if (request.method == 'PATCH') return http.Response(request.body, 200); + return http.Response(jsonEncode(data[key]), 200); + }); +} + +FirebaseRestClient _firebase(_SlowFirebase fake) => FirebaseRestClient( + databaseUrl: 'https://x-rtdb.europe-west1.firebasedatabase.app', + auth: FirebaseTokenProvider( + apiKey: 'AIzaKey', + store: InMemoryCredentialStore( + FirebaseCredentials( + idToken: 'id', + refreshToken: 'refresh', + expiresAt: DateTime.now().add(const Duration(hours: 1)), + ), + ), + ), + httpClient: fake.client(), +); + +void main() { + late NoteRepository repo; + + setUp(() async { + TestWidgetsFlutterBinding.ensureInitialized(); + installFakeSecureStorage(); + SharedPreferences.setMockInitialValues({}); + repo = await NoteRepository.openInMemory(nodeId: 'me'); + }); + + tearDown(() async => repo.close()); + + test('a Firebase-only sync clears the spinner in under 1 s', () async { + // The phone's setup: Firebase signed in, no GitHub token. + final fake = _SlowFirebase(); + final github = []; + final githubClient = http_testing.MockClient((request) async { + github.add('${request.method} ${request.url.path}'); + await Future.delayed(_rtt); + return http.Response('', 404); + }); + final state = InMemorySyncStateStore(); + const none = SyncSettings(owner: '', repo: '', token: ''); + Future once() => runSync( + repo, + none, + appSettings: const AppSettings(advancedMode: true), + httpClient: githubClient, + firebaseFactory: () async => _firebase(fake), + stateStore: state, + ); + + await once(); // first run publishes the header and our log + fake.requests.clear(); + github.clear(); + final clock = Stopwatch()..start(); + final result = await once(); + clock.stop(); + + expect(result.syncResult.firebaseConnected, isTrue); + expect(github, isEmpty, reason: 'no GitHub token: nothing to mirror'); + expect( + clock.elapsed, + lessThan(const Duration(seconds: 1)), + reason: 'firebase: ${fake.requests}', + ); + }); +} diff --git a/test/sync_service_pull_speed_test.dart b/test/sync_service_pull_speed_test.dart new file mode 100644 index 0000000..0ac4ba2 --- /dev/null +++ b/test/sync_service_pull_speed_test.dart @@ -0,0 +1,156 @@ +/// The pull half must not scale with peers one after another. +/// +/// The board reported a 3.9 s sync on the Pixel (todo:0c575b891b): eight +/// peers of about 2 MB each were fetched, decrypted, parsed and merged in +/// series, each merge rewriting the whole local store. +@TestOn('vm') +library; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:todo/data/note_repository.dart'; +import 'package:todo/sync/sync_service.dart'; + +import 'sync_service_fakes.dart'; + +/// A [FakeRemote] whose every call waits [delay] and tracks how many of +/// them are in flight at once. +class _SlowRemote extends FakeRemote { + _SlowRemote(super.files); + + static const delay = Duration(milliseconds: 50); + int _inFlight = 0; + int maxInFlight = 0; + + Future _slow(Future Function() run) async { + _inFlight++; + if (_inFlight > maxInFlight) maxInFlight = _inFlight; + try { + await Future.delayed(delay); + return await run(); + } finally { + _inFlight--; + } + } + + @override + Future getFileText(String path) => + _slow(() => super.getFileText(path)); + + @override + Future> listDirectory(String path) => + _slow(() => super.listDirectory(path)); + + @override + Future> getStringMap(String path) => + _slow(() => super.getStringMap(path)); +} + +/// A [FakeRemote] that cannot decrypt [undecryptable]. +class _LockedPeerRemote extends FakeRemote { + _LockedPeerRemote(super.files, this.undecryptable); + + final String undecryptable; + + @override + Future getFileText(String path) async { + if (path.endsWith(undecryptable)) throw E2eDecryptError('wrong key'); + return super.getFileText(path); + } +} + +/// Counts writes to the local store. +class _CountingPersistence extends MemoryPersistence { + int writes = 0; + + @override + Future write(String text) { + writes++; + return super.write(text); + } +} + +Future> _peers(int count) async { + final files = {}; + for (var i = 0; i < count; i++) { + final text = logToJson(await buildPeerLog(title: 'peer $i')); + files['todo-sync/notes/peer$i.json'] = text; + files['todo-sync/revs/peer$i'] = revisionOf(text); + } + return files; +} + +void main() { + test('fetches changed peers and the revision map concurrently', () async { + final remote = _SlowRemote(await _peers(4)); + final repo = await NoteRepository.openInMemory(nodeId: 'me'); + addTearDown(repo.close); + + final result = await SyncService( + stateStore: InMemorySyncStateStore(), + ).sync(repo, remote); + + expect(result.mergedDevices, 4); + expect(remote.maxInFlight, greaterThanOrEqualTo(4)); + }); + + test('merges every changed peer with one local store write', () async { + final remote = FakeRemote(await _peers(3)); + final persistence = _CountingPersistence(); + final repo = await NoteRepository.openWith( + persistence: persistence, + nodeId: 'me', + ); + addTearDown(repo.close); + persistence.writes = 0; + + final result = await SyncService( + stateStore: InMemorySyncStateStore(), + ).sync(repo, remote); + + expect(result.mergedDevices, 3); + expect(persistence.writes, 1); + expect(await repo.listNotes(), hasLength(1)); // same id on every peer + }); + + test('still skips an undecodable peer and merges the rest', () async { + final files = await _peers(2); + files['todo-sync/notes/bad.json'] = '{not json'; + final repo = await NoteRepository.openInMemory(nodeId: 'me'); + addTearDown(repo.close); + + final result = await SyncService( + stateStore: InMemorySyncStateStore(), + ).sync(repo, FakeRemote(files)); + + expect(result.mergedDevices, 2); + expect(result.skippedFiles, ['bad.json']); + }); + + test('skips a peer it cannot decrypt and merges the rest', () async { + final repo = await NoteRepository.openInMemory(nodeId: 'me'); + addTearDown(repo.close); + + final result = await SyncService( + stateStore: InMemorySyncStateStore(), + ).sync(repo, _LockedPeerRemote(await _peers(2), 'peer1.json')); + + expect(result.mergedDevices, 1); + expect(result.skippedFiles, ['peer1.json']); + }); + + test('a phone-sized peer log decodes on a background isolate', () async { + // Over the inline threshold, so this runs the isolate path end to end. + final big = logToJson(await buildPeerLog(title: 'x' * 100000)); + final repo = await NoteRepository.openInMemory(nodeId: 'me'); + addTearDown(repo.close); + + final result = await SyncService().sync( + repo, + FakeRemote({'todo-sync/notes/big.json': big}), + ); + + expect(result.mergedDevices, 1); + expect((await repo.listNotes()).single.text, 'x' * 100000); + }); +}