mirror of
https://github.com/kuhyx/utils.git
synced 2026-11-10 19:07:17 +01:00
Extracts the three-method remote surface GitHub was reached through into a `RemoteStore` interface, then implements it against Firebase Realtime Database over REST, so apps can move backend with a constructor change. Why REST and not the SDKs: official FlutterFire has no Linux desktop support, and `wake_alarm`'s PC side is a dependency-free systemd job. The RTDB REST endpoints are plain HTTPS and behave identically on Linux, Android and headless. Firebase is a transport, never the datastore -- each app keeps its local log as the source of truth, so offline-first survives. Why RTDB and not Firestore: on the Spark plan RTDB bills only storage and bandwidth with no per-operation quota, so a runaway loop cannot exhaust a daily budget and silently stop syncing mid-day. Two savings make the no-cost tier a guarantee rather than a hope. Measured against the GitHub-backed history this replaces: 4894 of 5542 commits (88.3%) were byte-identical no-ops, and every tick re-downloaded every peer's whole log. Both are now driven by one sha256 revision -- an unchanged log is not re-pushed, and an unchanged peer is not re-fetched. That turns egress from "ticks x log size" (~700 MB/month, and growing with the log) into "edits" (~40 MB/month against a 10 GiB cap). Revisions are one node per device, written only by its owner, rather than one shared map -- a whole-map write would erase every peer's entry, after which those peers would look permanently unchanged and never be fetched again. Per-device keys make that impossible rather than merely avoided. `.` is illegal in an RTDB key and the REST `.json` is a format suffix, so paths get a reversible `~XX` escape (`log.json` -> `log~2Ejson`). Lossy extension-stripping would break `todo`, whose listings return filenames. `MirrorStore` / `MirrorSyncClient` dual-writes during the cutover. Reads consult *both* backends: with primary-only reads a migrated device would never see an un-migrated peer's writes while still mirroring its own back -- silent one-directional convergence with no error raised. Writes are asymmetric on purpose: a primary failure fails the tick, a mirror failure is logged and survived. Auth is Firebase email/password over identitytoolkit REST, with mandatory refresh-token rotation. No service-account key anywhere: it would bypass the security rules and is trivially extractable from an APK. Verified live against a real database, not only mocks: sign-in and refresh, key escaping round-tripping byte-for-byte, an unchanged tick writing nothing, two devices converging, a revoked session throwing rather than returning null, and a log written by the Python client being read by the Dart one. No app is migrated by this commit; every app still runs on GitHub. Dart: 194 tests. Python: 203 tests at 100% branch coverage. No lint suppressions added.
362 lines
12 KiB
Dart
362 lines
12 KiB
Dart
/// Tests for the dual-write decorator used during the GitHub -> Firebase
|
|
/// cutover.
|
|
///
|
|
/// The asymmetry is the whole point and is what these assert: a primary
|
|
/// failure must fail the tick, a mirror failure must not, and reads must
|
|
/// consult both so that a half-migrated app (one device moved, one not) still
|
|
/// converges in both directions.
|
|
library;
|
|
|
|
import 'package:crdt_sync/crdt_sync.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
/// A scriptable in-memory [RemoteStore] that can be told to fail.
|
|
class _FakeStore implements RemoteStore, BulkMapReader {
|
|
_FakeStore({Map<String, String>? files, this.failing = false})
|
|
: files = files ?? {};
|
|
|
|
final Map<String, String> files;
|
|
bool failing;
|
|
final List<String> writes = [];
|
|
bool closed = false;
|
|
|
|
void _guard(String what) {
|
|
if (failing) throw RemoteSyncError('$what failed');
|
|
}
|
|
|
|
@override
|
|
Future<List<String>> listDirectory(String path) async {
|
|
_guard('list');
|
|
return files.keys
|
|
.where((key) => key.startsWith('$path/'))
|
|
.map((key) => key.substring(path.length + 1).split('/').first)
|
|
.toSet()
|
|
.toList();
|
|
}
|
|
|
|
@override
|
|
Future<String?> getFileText(String path) async {
|
|
_guard('read');
|
|
return files[path];
|
|
}
|
|
|
|
@override
|
|
Future<void> putFileText(
|
|
String path,
|
|
String text, {
|
|
required String message,
|
|
}) async {
|
|
_guard('write');
|
|
writes.add(path);
|
|
files[path] = text;
|
|
}
|
|
|
|
@override
|
|
Future<Map<String, String>> getStringMap(String path) async {
|
|
_guard('map read');
|
|
return {
|
|
for (final entry in files.entries)
|
|
if (entry.key.startsWith('$path/'))
|
|
entry.key.substring(path.length + 1): entry.value,
|
|
};
|
|
}
|
|
|
|
@override
|
|
Future<void> deleteFile(String path, {String message = ''}) async {
|
|
_guard('delete');
|
|
files.remove(path);
|
|
}
|
|
|
|
@override
|
|
Future<bool> canAccessRemote() async => !failing;
|
|
|
|
@override
|
|
void close() => closed = true;
|
|
}
|
|
|
|
/// A store with no bulk-map capability, standing in for [GitHubClient].
|
|
class _FakeStoreWithoutBulkRead implements RemoteStore {
|
|
_FakeStoreWithoutBulkRead([Map<String, String>? files])
|
|
: _inner = _FakeStore(files: files);
|
|
|
|
final _FakeStore _inner;
|
|
|
|
@override
|
|
Future<List<String>> listDirectory(String path) =>
|
|
_inner.listDirectory(path);
|
|
@override
|
|
Future<String?> getFileText(String path) => _inner.getFileText(path);
|
|
@override
|
|
Future<void> putFileText(String p, String t, {required String message}) =>
|
|
_inner.putFileText(p, t, message: message);
|
|
@override
|
|
Future<void> deleteFile(String path, {String message = ''}) =>
|
|
_inner.deleteFile(path, message: message);
|
|
@override
|
|
Future<bool> canAccessRemote() => _inner.canAccessRemote();
|
|
@override
|
|
void close() => _inner.close();
|
|
}
|
|
|
|
({MirrorStore store, _FakeStore primary, _FakeStore mirror, List<String> failures})
|
|
_mirror({
|
|
Map<String, String>? primaryFiles,
|
|
Map<String, String>? mirrorFiles,
|
|
bool primaryFailing = false,
|
|
bool mirrorFailing = false,
|
|
}) {
|
|
final primary = _FakeStore(files: primaryFiles, failing: primaryFailing);
|
|
final mirror = _FakeStore(files: mirrorFiles, failing: mirrorFailing);
|
|
final failures = <String>[];
|
|
return (
|
|
store: MirrorStore(
|
|
primary: primary,
|
|
mirror: mirror,
|
|
onMirrorFailure: (operation, _) => failures.add(operation),
|
|
),
|
|
primary: primary,
|
|
mirror: mirror,
|
|
failures: failures,
|
|
);
|
|
}
|
|
|
|
void main() {
|
|
group('writes', () {
|
|
test('go to both backends', () async {
|
|
final m = _mirror();
|
|
await m.store.putFileText('ns/pc/log.json', '{}', message: 'm');
|
|
expect(m.primary.writes, ['ns/pc/log.json']);
|
|
expect(m.mirror.writes, ['ns/pc/log.json']);
|
|
});
|
|
|
|
test('a primary failure fails the tick', () async {
|
|
// Fail-closed: the primary is authoritative, so a sync that could not
|
|
// write it must not be reported as successful.
|
|
final m = _mirror(primaryFailing: true);
|
|
await expectLater(
|
|
() => m.store.putFileText('ns/pc/log.json', '{}', message: 'm'),
|
|
throwsA(isA<RemoteSyncError>()),
|
|
);
|
|
});
|
|
|
|
test('a mirror failure is loud but does not fail the tick', () async {
|
|
final m = _mirror(mirrorFailing: true);
|
|
await m.store.putFileText('ns/pc/log.json', '{}', message: 'm');
|
|
expect(m.primary.writes, ['ns/pc/log.json']);
|
|
expect(m.failures, ['putFileText ns/pc/log.json']);
|
|
});
|
|
|
|
test('deletes behave the same way', () async {
|
|
final m = _mirror(
|
|
primaryFiles: {'ns/pc/log.json': '{}'},
|
|
mirrorFiles: {'ns/pc/log.json': '{}'},
|
|
mirrorFailing: true,
|
|
);
|
|
await m.store.deleteFile('ns/pc/log.json');
|
|
expect(m.primary.files, isEmpty);
|
|
expect(m.failures, ['deleteFile ns/pc/log.json']);
|
|
});
|
|
|
|
test('a primary delete failure fails the tick', () async {
|
|
final m = _mirror(primaryFailing: true);
|
|
await expectLater(
|
|
() => m.store.deleteFile('ns/pc/log.json'),
|
|
throwsA(isA<RemoteSyncError>()),
|
|
);
|
|
});
|
|
});
|
|
|
|
group('reads consult both backends', () {
|
|
test('prefer the primary when it has the file', () async {
|
|
final m = _mirror(
|
|
primaryFiles: {'ns/pc/log.json': 'from-primary'},
|
|
mirrorFiles: {'ns/pc/log.json': 'from-mirror'},
|
|
);
|
|
expect(await m.store.getFileText('ns/pc/log.json'), 'from-primary');
|
|
});
|
|
|
|
test('fall back to the mirror for an un-migrated device', () async {
|
|
// This is why reads are not primary-only: a migrated desktop must
|
|
// still see an un-migrated phone's writes, or convergence silently
|
|
// becomes one-directional with no error raised.
|
|
final m = _mirror(mirrorFiles: {'ns/phone/log.json': 'from-mirror'});
|
|
expect(await m.store.getFileText('ns/phone/log.json'), 'from-mirror');
|
|
});
|
|
|
|
test('return null when neither backend has the file', () async {
|
|
final m = _mirror();
|
|
expect(await m.store.getFileText('ns/nobody/log.json'), isNull);
|
|
});
|
|
|
|
test('a mirror read failure degrades to the primary answer', () async {
|
|
final m = _mirror(mirrorFailing: true);
|
|
expect(await m.store.getFileText('ns/pc/log.json'), isNull);
|
|
expect(m.failures, ['getFileText ns/pc/log.json']);
|
|
});
|
|
|
|
test('a primary read failure fails the tick', () async {
|
|
final m = _mirror(primaryFailing: true);
|
|
await expectLater(
|
|
() => m.store.getFileText('ns/pc/log.json'),
|
|
throwsA(isA<RemoteSyncError>()),
|
|
);
|
|
});
|
|
});
|
|
|
|
group('listDirectory', () {
|
|
test('unions devices from both backends', () async {
|
|
final m = _mirror(
|
|
primaryFiles: {'ns/pc/log.json': '{}'},
|
|
mirrorFiles: {'ns/phone/log.json': '{}'},
|
|
);
|
|
expect(await m.store.listDirectory('ns'), containsAll(['pc', 'phone']));
|
|
});
|
|
|
|
test('does not duplicate a device present in both', () async {
|
|
final m = _mirror(
|
|
primaryFiles: {'ns/pc/log.json': '{}'},
|
|
mirrorFiles: {'ns/pc/log.json': '{}'},
|
|
);
|
|
expect(await m.store.listDirectory('ns'), ['pc']);
|
|
});
|
|
|
|
test('a mirror failure degrades to the primary list', () async {
|
|
final m = _mirror(
|
|
primaryFiles: {'ns/pc/log.json': '{}'},
|
|
mirrorFailing: true,
|
|
);
|
|
expect(await m.store.listDirectory('ns'), ['pc']);
|
|
expect(m.failures, ['listDirectory ns']);
|
|
});
|
|
|
|
test('a primary failure fails the tick', () async {
|
|
final m = _mirror(primaryFailing: true);
|
|
await expectLater(
|
|
() => m.store.listDirectory('ns'),
|
|
throwsA(isA<RemoteSyncError>()),
|
|
);
|
|
});
|
|
});
|
|
|
|
group('revision maps', () {
|
|
test('merge both backends, primary winning on conflict', () async {
|
|
final m = _mirror(
|
|
primaryFiles: {'ns/revs/pc': 'primary-rev'},
|
|
mirrorFiles: {'ns/revs/pc': 'mirror-rev', 'ns/revs/phone': 'old-rev'},
|
|
);
|
|
expect(await m.store.getStringMap('ns/revs'), {
|
|
'pc': 'primary-rev',
|
|
// An un-migrated device publishes revisions only to the mirror;
|
|
// without this it would look revision-less and be re-downloaded
|
|
// every tick for the whole trial.
|
|
'phone': 'old-rev',
|
|
});
|
|
});
|
|
|
|
test('a mirror failure degrades to primary revisions only', () async {
|
|
final m = _mirror(
|
|
primaryFiles: {'ns/revs/pc': 'primary-rev'},
|
|
mirrorFailing: true,
|
|
);
|
|
expect(await m.store.getStringMap('ns/revs'), {'pc': 'primary-rev'});
|
|
expect(m.failures, ['getStringMap ns/revs']);
|
|
});
|
|
|
|
test('a backend without bulk reads simply contributes nothing', () async {
|
|
final store = MirrorStore(
|
|
primary: _FakeStore(files: {'ns/revs/pc': 'primary-rev'}),
|
|
mirror: _FakeStoreWithoutBulkRead({'ns/revs/phone': 'ignored'}),
|
|
);
|
|
expect(await store.getStringMap('ns/revs'), {'pc': 'primary-rev'});
|
|
});
|
|
|
|
test('is empty when the primary has no bulk reads either', () async {
|
|
final store = MirrorStore(
|
|
primary: _FakeStoreWithoutBulkRead(),
|
|
mirror: _FakeStoreWithoutBulkRead(),
|
|
);
|
|
expect(await store.getStringMap('ns/revs'), isEmpty);
|
|
});
|
|
});
|
|
|
|
group('lifecycle', () {
|
|
test('canAccessRemote reports only the primary', () async {
|
|
// A Test-connection button must not report success because the backend
|
|
// being retired happens to answer.
|
|
final m = _mirror(primaryFailing: true);
|
|
expect(await m.store.canAccessRemote(), isFalse);
|
|
final healthy = _mirror(mirrorFailing: true);
|
|
expect(await healthy.store.canAccessRemote(), isTrue);
|
|
});
|
|
|
|
test('close releases both backends', () {
|
|
final m = _mirror();
|
|
m.store.close();
|
|
expect(m.primary.closed, isTrue);
|
|
expect(m.mirror.closed, isTrue);
|
|
});
|
|
|
|
test('a mirror failure with no handler is still swallowed', () async {
|
|
final store = MirrorStore(
|
|
primary: _FakeStore(),
|
|
mirror: _FakeStore(failing: true),
|
|
);
|
|
await expectLater(
|
|
store.putFileText('ns/pc/log.json', '{}', message: 'm'),
|
|
completes,
|
|
);
|
|
});
|
|
|
|
test('is itself a RemoteStore, so syncLog accepts it', () {
|
|
expect(_mirror().store, isA<RemoteStore>());
|
|
});
|
|
});
|
|
|
|
group('end to end through syncLog', () {
|
|
test('a half-migrated pair still converges both ways', () async {
|
|
// desktop is on Firebase+mirror; phone has not migrated and still
|
|
// writes only to GitHub. Both must end up with both records.
|
|
final github = _FakeStore();
|
|
final firebase = _FakeStore();
|
|
|
|
Log log(String id, String nodeId) => {
|
|
id: Record(
|
|
id: id,
|
|
fields: {
|
|
'v': (id, Hlc(wallTimeMs: 1000, counter: 0, nodeId: nodeId)),
|
|
},
|
|
),
|
|
};
|
|
// The library's own serialization, so this exercises the same
|
|
// encoding the apps use rather than a test-only one.
|
|
const encode = logToJson;
|
|
const decode = logFromJson;
|
|
|
|
// Un-migrated phone: GitHub only.
|
|
await syncLog(
|
|
client: github,
|
|
deviceId: 'phone',
|
|
pathPrefix: 'ns/devices',
|
|
localLog: log('from-phone', 'node-phone'),
|
|
encode: encode,
|
|
decode: decode,
|
|
);
|
|
|
|
// Migrated desktop: Firebase primary, GitHub mirror.
|
|
final merged = await syncLog(
|
|
client: MirrorStore(primary: firebase, mirror: github),
|
|
deviceId: 'desktop',
|
|
pathPrefix: 'ns/devices',
|
|
localLog: log('from-desktop', 'node-desktop'),
|
|
encode: encode,
|
|
decode: decode,
|
|
);
|
|
|
|
expect(merged.keys, containsAll(['from-phone', 'from-desktop']));
|
|
// The desktop's merged result is mirrored back to GitHub, so the
|
|
// un-migrated phone sees it on its next tick.
|
|
expect(github.files['ns/devices/desktop/log.json'], isNotNull);
|
|
});
|
|
});
|
|
}
|