diff --git a/crdt-sync/crdt_sync/__init__.py b/crdt-sync/crdt_sync/__init__.py index 031b85c..61f0861 100644 --- a/crdt-sync/crdt_sync/__init__.py +++ b/crdt-sync/crdt_sync/__init__.py @@ -1,28 +1,70 @@ -"""Shared CRDT merge scheme + GitHub-Contents-API sync transport.""" +"""Shared CRDT merge scheme + pluggable remote sync transports.""" from __future__ import annotations +from crdt_sync._firebase import ( + DatabaseNotFoundError, + FirebaseSyncClient, + FirebaseSyncError, +) +from crdt_sync._firebase_auth import ( + CredentialStore, + FileCredentialStore, + FirebaseAuthError, + FirebaseCredentials, + FirebaseTokenProvider, + MemoryCredentialStore, +) from crdt_sync._github import GitHubSyncClient, GitHubSyncError, RepoNotFoundError from crdt_sync._hlc import Hlc from crdt_sync._log import Log, merge_logs +from crdt_sync._mirror import MirrorSyncClient from crdt_sync._record import Field, Record, merge_field, merge_record +from crdt_sync._remote import RemoteNotFoundError, RemoteStore, RemoteSyncError from crdt_sync._store import dump_log, load_log, read_log, write_log -from crdt_sync._sync import sync_log +from crdt_sync._sync import ( + FileSyncStateStore, + MemorySyncStateStore, + SyncState, + SyncStateStore, + default_revs_path, + revision_of, + sync_log, +) __all__ = [ + "CredentialStore", + "DatabaseNotFoundError", "Field", + "FileCredentialStore", + "FileSyncStateStore", + "FirebaseAuthError", + "FirebaseCredentials", + "FirebaseSyncClient", + "FirebaseSyncError", + "FirebaseTokenProvider", "GitHubSyncClient", "GitHubSyncError", "Hlc", "Log", + "MemoryCredentialStore", + "MemorySyncStateStore", + "MirrorSyncClient", "Record", + "RemoteNotFoundError", + "RemoteStore", + "RemoteSyncError", "RepoNotFoundError", + "SyncState", + "SyncStateStore", + "default_revs_path", "dump_log", "load_log", "merge_field", "merge_logs", "merge_record", "read_log", + "revision_of", "sync_log", "write_log", ] diff --git a/crdt-sync/crdt_sync/_firebase.py b/crdt-sync/crdt_sync/_firebase.py new file mode 100644 index 0000000..44b7a36 --- /dev/null +++ b/crdt-sync/crdt_sync/_firebase.py @@ -0,0 +1,330 @@ +"""Firebase Realtime Database as dumb keyed storage, over its REST API. + +The RTDB REST endpoints are plain HTTPS, so this works identically under +systemd, in a venv, and alongside the Dart client on Android -- and the two +write byte-identical payloads, so a device on either implementation reads the +other's. + +Why RTDB rather than Firestore: on the Spark (free) plan RTDB bills only +storage and bandwidth, with **no per-operation quota**, so a misbehaving sync +loop can never exhaust a daily budget and silently stop working mid-day. + +Mirrors ``crdt_sync_dart``'s ``lib/src/firebase_client.dart``; keep the two in +step. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import requests + +from crdt_sync._remote import RemoteNotFoundError, RemoteSyncError + +if TYPE_CHECKING: + from crdt_sync._firebase_auth import FirebaseTokenProvider + +_DEFAULT_TIMEOUT_SECONDS = 15 +_HTTP_UNAUTHORIZED = 401 +_HTTP_FORBIDDEN = 403 + +# Characters Realtime Database forbids in a key, plus ``~`` itself because it +# is this module's escape character. ``/`` is absent deliberately: it is the +# path separator, handled by splitting into segments before escaping. +_ESCAPES = { + "~": "~7E", + ".": "~2E", + "$": "~24", + "#": "~23", + "[": "~5B", + "]": "~5D", +} +_UNESCAPES = {escape: char for char, escape in _ESCAPES.items()} +_ESCAPE_LENGTH = 3 + + +class FirebaseSyncError(RemoteSyncError): + """Raised for an RTDB failure the caller must not silently ignore.""" + + +class DatabaseNotFoundError(FirebaseSyncError, RemoteNotFoundError): + """Raised when the database is unreachable or the uid is not allowed. + + The Firebase counterpart of :class:`crdt_sync.RepoNotFoundError`: "the + database URL is wrong, or the security rules reject this account", as + opposed to "nothing has been pushed to that path yet", which is benign + and surfaces as ``None``. + """ + + +def encode_key(segment: str) -> str: + """Escape one path segment into a legal Realtime Database key. + + RTDB rejects ``. $ # [ ] /`` in keys, and the REST API's trailing + ``.json`` is a *format suffix* rather than part of the path -- so a + filename like ``log.json`` cannot be stored verbatim. The mapping is a + reversible ``~XX`` escape (``log.json`` -> ``log~2Ejson``) rather than a + lossy "strip the extension", because callers see these names again: + todo-app lists *filenames*, not device directories, and must keep getting + ``.json`` back. + + ``~`` is the escape character because it is legal in RTDB keys and + unreserved in URLs, so the escaped form needs no percent-encoding -- + which the server would otherwise decode back into the illegal character. + + Parameters + ---------- + segment : str + One path segment, with no ``/`` in it. + + Returns: + ------- + str + The escaped key. + + """ + return "".join(_ESCAPES.get(char, char) for char in segment) + + +def decode_key(key: str) -> str: + """Reverse :func:`encode_key`. + + Parameters + ---------- + key : str + An escaped Realtime Database key. + + Returns: + ------- + str + The original segment. + + """ + out: list[str] = [] + index = 0 + while index < len(key): + escape = key[index : index + _ESCAPE_LENGTH] + char = _UNESCAPES.get(escape) + if char is None: + out.append(key[index]) + index += 1 + else: + out.append(char) + index += _ESCAPE_LENGTH + return "".join(out) + + +def encode_path(path: str) -> str: + """Escape every segment of a ``/``-separated logical path. + + Parameters + ---------- + path : str + A logical path such as ``"todo-sync/notes/abc.json"``. + + Returns: + ------- + str + The path with each segment escaped, empty segments dropped. + + """ + return "/".join(encode_key(part) for part in path.split("/") if part) + + +class FirebaseSyncClient: + """Realtime Database seen through the same contract as GitHub. + + Reads and writes UTF-8 text blobs stored as JSON string leaves, so the + payload is byte-identical to what the GitHub backend stored and the two + can be mirrored against each other during migration. + """ + + def __init__( + self, + database_url: str, + auth: FirebaseTokenProvider, + *, + timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS, + ) -> None: + """Create a client for ``database_url`` authenticated by ``auth``. + + Args: + database_url: The database's HTTPS origin. + auth: Mints the ID token every request carries. + timeout_seconds: Per-request network timeout. + """ + self._database_url = database_url.rstrip("/") + self.auth = auth + self._timeout_seconds = timeout_seconds + + def _url(self, path: str) -> str: + return f"{self._database_url}/{encode_path(path)}.json" + + def _params(self, **extra: str) -> dict[str, str]: + return {"auth": self.auth.id_token(), **extra} + + def _raise(self, what: str, response: requests.Response) -> None: + """Raise the right error type for a non-2xx response. + + 401/403 mean the rules rejected this uid or the token is bad. + Everything else -- including a Spark quota exhaustion, which answers + with an error rather than billing you -- is a plain + :class:`FirebaseSyncError`. Nothing is ever swallowed: a + quota-exhausted database that silently returned "no data" would look + exactly like "nothing to sync". + """ + detail = response.text.strip() + if response.status_code in (_HTTP_UNAUTHORIZED, _HTTP_FORBIDDEN): + msg = ( + f"{what} rejected: HTTP {response.status_code} {detail} -- " + f"the database URL or the security rules do not allow this " + f"account" + ) + raise DatabaseNotFoundError(msg) + msg = f"{what} failed: HTTP {response.status_code} {detail}" + raise FirebaseSyncError(msg) + + def _get(self, path: str, **params: str) -> requests.Response: + try: + return requests.get( + self._url(path), + params=self._params(**params), + timeout=self._timeout_seconds, + ) + except requests.RequestException as exc: + msg = f"network error reading {path}" + raise FirebaseSyncError(msg) from exc + + def list_directory(self, path: str) -> list[str]: + """Return the entry names directly under ``path``. + + Uses ``shallow=true``, which returns ``{key: true}`` without any + values -- so a listing costs bytes rather than the whole subtree. + That is the difference between a sync tick costing hundreds of bytes + and hundreds of kilobytes. + + Raises: + DatabaseNotFoundError: If the rules reject this account. + FirebaseSyncError: For any other non-2xx response or network + error. + """ + response = self._get(path, shallow="true") + if not response.ok: + self._raise(f"listing {path}", response) + data = response.json() + if not isinstance(data, dict): + return [] + return [decode_key(key) for key in data] + + def get_file_text(self, path: str) -> str | None: + """Return the text stored at ``path``, or ``None`` if absent. + + Raises: + DatabaseNotFoundError: If the rules reject this account. + FirebaseSyncError: If the value is not a text blob, or for any + other non-2xx response or network error. + """ + response = self._get(path) + if not response.ok: + self._raise(f"reading {path}", response) + data = response.json() + if data is None: + return None + if not isinstance(data, str): + msg = ( + f"value at {path} is {type(data).__name__}, not the expected text blob" + ) + raise FirebaseSyncError(msg) + return data + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Write ``text`` at ``path``. + + Args: + path: A logical path such as ``"ns/devices/pc/log.json"``. + text: The full new content. + message: Ignored -- RTDB has no commit log to attach a reason to. + Present so this matches the ``RemoteStore`` contract. + + Raises: + FirebaseSyncError: On any non-2xx response or network error. + """ + del message + try: + response = requests.put( + self._url(path), + params=self._params(), + json=text, + timeout=self._timeout_seconds, + ) + except requests.RequestException as exc: + msg = f"network error writing {path}" + raise FirebaseSyncError(msg) from exc + if not response.ok: + self._raise(f"writing {path}", response) + + def get_string_map(self, path: str) -> dict[str, str]: + """Return the map at ``path`` as ``key -> text``, empty if absent. + + Tolerates a non-map or malformed value rather than raising: this + backs the revision cache, which is an *optimisation*. A corrupt revs + node must degrade into "fetch everything", never into a failed sync. + + Raises: + FirebaseSyncError: Only for a transport-level failure. + """ + response = self._get(path) + if not response.ok: + self._raise(f"reading {path}", response) + data = response.json() + if not isinstance(data, dict): + return {} + return { + decode_key(key): value + for key, value in data.items() + if isinstance(value, str) + } + + def delete_file(self, path: str, *, message: str = "crdt_sync: delete") -> None: + """Delete ``path``. A no-op if it does not exist. + + Args: + path: A logical path. + message: Ignored, as for :meth:`put_file_text`. + + Raises: + FirebaseSyncError: On any non-2xx response or network error. + """ + del message + try: + response = requests.delete( + self._url(path), + params=self._params(), + timeout=self._timeout_seconds, + ) + except requests.RequestException as exc: + msg = f"network error deleting {path}" + raise FirebaseSyncError(msg) from exc + if not response.ok: + self._raise(f"deleting {path}", response) + + def can_access_remote(self) -> bool: + """Return whether the credential can reach the database. + + Never raises -- a bad URL, a rejected token, a missing session or a + network failure all report ``False``, so a settings "Test connection" + button cannot blow up. + """ + try: + response = requests.get( + self._url(""), + params=self._params(shallow="true"), + timeout=self._timeout_seconds, + ) + except requests.RequestException: + return False + except RemoteSyncError: + # Includes FirebaseAuthError: "cannot get a token" is exactly + # "cannot access the remote". + return False + return response.ok diff --git a/crdt-sync/crdt_sync/_firebase_auth.py b/crdt-sync/crdt_sync/_firebase_auth.py new file mode 100644 index 0000000..85c22d0 --- /dev/null +++ b/crdt-sync/crdt_sync/_firebase_auth.py @@ -0,0 +1,325 @@ +"""Firebase Authentication over its REST API, with no SDK dependency. + +The Firebase Admin SDK authenticates with a service-account key, which +bypasses security rules entirely and would have to be distributed to every +device. These are single-user personal apps, so devices instead sign in as +one ordinary user and hold a refresh token -- the same path the Flutter side +takes, so both platforms behave identically. + +Mirrors ``crdt_sync_dart``'s ``lib/src/firebase_auth_rest.dart``; keep the +two in step. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import datetime as dt +import json +from typing import TYPE_CHECKING, Protocol + +import requests + +from crdt_sync._remote import RemoteSyncError + +if TYPE_CHECKING: + from pathlib import Path + +_SIGN_IN_BASE = "https://identitytoolkit.googleapis.com/v1" +_REFRESH_BASE = "https://securetoken.googleapis.com/v1" +_DEFAULT_TIMEOUT_SECONDS = 15 + +# Refresh this long before the token actually expires. An ID token lives ~1h; +# refreshing early means a tick that starts just under the wire still finishes +# with a valid token rather than 401ing halfway through a multi-file push. +_REFRESH_SKEW = dt.timedelta(minutes=5) + + +class FirebaseAuthError(RemoteSyncError): + """Raised for an authentication failure the caller must not ignore. + + A :class:`crdt_sync.RemoteSyncError` so callers that only care about + "sync is broken" can catch one type, while a settings screen can single + this out to say "your password is wrong" rather than "the network is + down". + """ + + +@dataclass(frozen=True) +class FirebaseCredentials: + """One device's session: a short-lived ID token plus its refresh token. + + Attributes: + ---------- + id_token: + The bearer credential for Realtime Database requests. Expires fast. + refresh_token: + The long-lived credential. **This is the secret worth protecting.** + expires_at: + When ``id_token`` stops being accepted, in UTC. + """ + + id_token: str + refresh_token: str + expires_at: dt.datetime + + def is_expired_at(self, now: dt.datetime) -> bool: + """Return whether the token is expired, or close enough to it.""" + return now + _REFRESH_SKEW >= self.expires_at + + def to_json(self) -> dict[str, str]: + """Return a JSON-serializable form.""" + return { + "id_token": self.id_token, + "refresh_token": self.refresh_token, + "expires_at": self.expires_at.isoformat(), + } + + @classmethod + def from_json(cls, data: dict[str, str]) -> FirebaseCredentials: + """Rebuild credentials from :meth:`to_json` output.""" + return cls( + id_token=data["id_token"], + refresh_token=data["refresh_token"], + expires_at=dt.datetime.fromisoformat(data["expires_at"]), + ) + + +class CredentialStore(Protocol): + """Where a device keeps its credentials between runs.""" + + def load(self) -> FirebaseCredentials | None: + """Return the stored credentials, or ``None`` if not signed in.""" + + def save(self, credentials: FirebaseCredentials) -> None: + """Persist ``credentials``.""" + + def clear(self) -> None: + """Forget any stored credentials.""" + + +class MemoryCredentialStore: + """A :class:`CredentialStore` for tests and one-shot scripts.""" + + def __init__(self, credentials: FirebaseCredentials | None = None) -> None: + """Start holding ``credentials`` (or nothing).""" + self._credentials = credentials + + def load(self) -> FirebaseCredentials | None: + """Return the held credentials.""" + return self._credentials + + def save(self, credentials: FirebaseCredentials) -> None: + """Replace the held credentials.""" + self._credentials = credentials + + def clear(self) -> None: + """Drop the held credentials.""" + self._credentials = None + + +class FileCredentialStore: + """A :class:`CredentialStore` backed by a ``0600`` JSON file. + + Persistence is not an optimisation here: ``wake_alarm``'s PC side is a + fresh process every minute and ``diet_guard``'s every 15 minutes, so an + in-memory store would re-authenticate thousands of times a day. + """ + + def __init__(self, path: Path) -> None: + """Store credentials at ``path``, creating parent dirs as needed.""" + self._path = path + + def load(self) -> FirebaseCredentials | None: + """Return the stored credentials, or ``None`` if the file is absent. + + A corrupt or truncated file (an interrupted write) reads as "not + signed in" rather than raising: the caller's next step is to sign in + again, which repairs it. + """ + try: + data = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + try: + return FirebaseCredentials.from_json(data) + except (KeyError, TypeError, ValueError): + return None + + def save(self, credentials: FirebaseCredentials) -> None: + """Write ``credentials`` atomically, readable only by this user.""" + self._path.parent.mkdir(parents=True, exist_ok=True) + temp = self._path.with_suffix(f"{self._path.suffix}.tmp") + # Create with 0600 from the outset -- writing then chmod'ing would + # leave the refresh token world-readable for the gap in between. + temp.touch(mode=0o600) + temp.write_text(json.dumps(credentials.to_json()), encoding="utf-8") + temp.replace(self._path) + + def clear(self) -> None: + """Delete the credentials file if it exists.""" + self._path.unlink(missing_ok=True) + + +def _utcnow() -> dt.datetime: + return dt.datetime.now(dt.UTC) + + +class FirebaseTokenProvider: + """Signs in and keeps a valid ID token available, refreshing as needed. + + ``api_key`` is the project's public Web API key -- not a secret, and safe + to ship inside an APK. The actual credential is the refresh token held by + the store. + """ + + def __init__( + self, + api_key: str, + store: CredentialStore, + *, + timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS, + clock: object = None, + ) -> None: + """Create a provider for ``api_key`` persisting through ``store``. + + Args: + api_key: The project's public Web API key. + store: Where the refresh token lives between runs. + timeout_seconds: Per-request network timeout. + clock: A zero-arg callable returning "now" (tests inject one). + """ + self._api_key = api_key + self._store = store + self._timeout_seconds = timeout_seconds + self._clock = clock if callable(clock) else _utcnow + self._cached: FirebaseCredentials | None = None + + def sign_in(self, email: str, password: str) -> None: + """Exchange an email/password for a session and persist it. + + Called once per device from a setup command; every later run reuses + the stored refresh token. + + Raises: + FirebaseAuthError: If the credentials are rejected. + """ + body = self._post( + f"{_SIGN_IN_BASE}/accounts:signInWithPassword?key={self._api_key}", + {"email": email, "password": password, "returnSecureToken": True}, + "sign in", + ) + self._adopt( + id_token=body["idToken"], + refresh_token=body["refreshToken"], + expires_in_seconds=body["expiresIn"], + ) + + def id_token(self) -> str: + """Return a currently-valid ID token, refreshing if needed. + + Never returns a stale token and never silently no-ops: a sync that + quietly stops syncing is the failure mode this design exists to + prevent. + + Raises: + FirebaseAuthError: If no session is stored, or the refresh token + has been revoked. + """ + credentials = self._load() + if credentials is None: + msg = "not signed in: no stored refresh token for this device" + raise FirebaseAuthError(msg) + if not credentials.is_expired_at(self._clock()): + return credentials.id_token + return self._refresh(credentials.refresh_token) + + def has_session(self) -> bool: + """Return whether this device has a stored session at all. + + Distinguishes "sync is not configured" -- a normal state for + ``screen-locker`` -- from "sync is configured and broken", an error. + """ + return self._load() is not None + + def sign_out(self) -> None: + """Forget the stored session, so the next token request fails.""" + self._cached = None + self._store.clear() + + def _load(self) -> FirebaseCredentials | None: + if self._cached is None: + self._cached = self._store.load() + return self._cached + + def _refresh(self, refresh_token: str) -> str: + body = self._post( + f"{_REFRESH_BASE}/token?key={self._api_key}", + {"grant_type": "refresh_token", "refresh_token": refresh_token}, + "refresh the session", + ) + return self._adopt( + id_token=body["id_token"], + # A refresh may hand back a rotated token; keep the new one. + refresh_token=body["refresh_token"], + expires_in_seconds=body["expires_in"], + ) + + def _adopt( + self, + *, + id_token: str, + refresh_token: str, + expires_in_seconds: str, + ) -> str: + credentials = FirebaseCredentials( + id_token=id_token, + refresh_token=refresh_token, + expires_at=self._clock() + dt.timedelta(seconds=int(expires_in_seconds)), + ) + self._cached = credentials + self._store.save(credentials) + return id_token + + def _post( + self, + url: str, + payload: dict[str, object], + what: str, + ) -> dict[str, str]: + try: + response = requests.post( + url, + json=payload, + timeout=self._timeout_seconds, + ) + except requests.RequestException as exc: + msg = f"network error trying to {what}" + raise FirebaseAuthError(msg) from exc + if not response.ok: + reason = _reason(response) + msg = f"failed to {what}: HTTP {response.status_code}{reason}" + raise FirebaseAuthError(msg) + return response.json() + + +def _reason(response: requests.Response) -> str: + """Return Google's machine-readable error reason, in parentheses. + + Pulls ``INVALID_PASSWORD`` / ``TOKEN_EXPIRED`` / ``USER_DISABLED`` out of + the body so the raised message says what actually went wrong rather than + just reporting a status code. + """ + try: + data = response.json() + except ValueError: + # A non-JSON body (a proxy error page, say): the status code is all + # the detail there is. + return "" + if not isinstance(data, dict): + return "" + error = data.get("error") + if isinstance(error, dict): + return f" ({error.get('message')})" + if isinstance(error, str): + return f" ({error})" + return "" diff --git a/crdt-sync/crdt_sync/_github.py b/crdt-sync/crdt_sync/_github.py index 18e15e7..8be178b 100644 --- a/crdt-sync/crdt_sync/_github.py +++ b/crdt-sync/crdt_sync/_github.py @@ -13,22 +13,28 @@ import base64 import requests +from crdt_sync._remote import RemoteNotFoundError, RemoteSyncError + _API_BASE = "https://api.github.com" _HTTP_NOT_FOUND = 404 _DEFAULT_TIMEOUT_SECONDS = 15 -class GitHubSyncError(Exception): +class GitHubSyncError(RemoteSyncError): """Raised for a GitHub API failure the caller must not silently ignore.""" -class RepoNotFoundError(GitHubSyncError): +class RepoNotFoundError(GitHubSyncError, RemoteNotFoundError): """Raised when the configured repo itself is unreachable. Distinguished from a path-404 (nothing pushed to that path yet, which is benign -- it just means no other device has synced before) so the caller can tell "the repo name is wrong or the token isn't scoped to it" apart from "no other device has synced yet". + + Derives from both so existing ``except GitHubSyncError`` handlers still + cover it *and* backend-neutral callers can catch "the remote itself is + missing" without naming GitHub. """ @@ -199,6 +205,17 @@ class GitHubSyncClient: Hits the bare repo endpoint, so it succeeds even before any file has been pushed. Never raises -- a network failure or missing repo is reported as ``False``. + + The GitHub-era name for :meth:`can_access_remote`; kept because app + settings screens call it. Both do the same probe. + """ + return self._repo_exists() + + def can_access_remote(self) -> bool: + """Return whether the token can reach the remote (a connection test). + + The :class:`crdt_sync.RemoteStore` spelling of + :meth:`can_access_repo`. """ return self._repo_exists() diff --git a/crdt-sync/crdt_sync/_mirror.py b/crdt-sync/crdt_sync/_mirror.py new file mode 100644 index 0000000..49931b0 --- /dev/null +++ b/crdt-sync/crdt_sync/_mirror.py @@ -0,0 +1,155 @@ +"""A remote store that writes to two backends and reads from both. + +Exists for the GitHub -> Firebase cutover: each app can move to Firebase +while still mirroring to the old repo, so rolling back is a constructor +change rather than a data recovery. + +Mirrors ``crdt_sync_dart``'s ``lib/src/mirror_store.dart``; keep the two in +step. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, TypeVar + +from crdt_sync._remote import RemoteSyncError + +if TYPE_CHECKING: + from collections.abc import Callable + + from crdt_sync._remote import RemoteStore + +_logger = logging.getLogger(__name__) + +# PEP 695 type parameters need Python 3.12; this package targets 3.10. +_T = TypeVar("_T") + + +class MirrorSyncClient: + """Writes to both backends; reads from both, preferring the primary. + + **Reads consult both on purpose.** Each app spans two devices, and they + cut over one at a time. If reads were primary-only, a migrated PC would + never see an un-migrated phone's writes while still mirroring its own + back -- silent one-directional convergence, with no error raised. Because + :func:`crdt_sync.merge_logs` is commutative and idempotent, a union read + costs nothing semantically and makes the cutover order irrelevant. + + Write asymmetry is deliberate too: the primary is authoritative, so a + primary failure fails the tick (fail-closed), while a mirror failure is + reported and otherwise tolerated. Once the mirror is retired, the old + backend going away must not break sync. + """ + + def __init__( + self, + primary: RemoteStore, + mirror: RemoteStore, + *, + on_mirror_failure: Callable[[str, Exception], None] | None = None, + ) -> None: + """Wrap ``primary`` (authoritative) and ``mirror`` (best effort). + + Args: + primary: The backend whose failures fail the tick. + mirror: The backend kept in step but never allowed to fail a tick. + on_mirror_failure: Notified when the mirror misbehaves. Defaults + to a warning log -- a silent mirror failure would let the + fallback rot unnoticed until the day it was needed. + """ + self.primary = primary + self.mirror = mirror + self._on_mirror_failure = on_mirror_failure or _log_mirror_failure + + def list_directory(self, path: str) -> list[str]: + """Return the union of both backends' entries under ``path``.""" + names = list(self.primary.list_directory(path)) + seen = set(names) + for name in ( + self._try_mirror( + f"list_directory {path}", lambda: self.mirror.list_directory(path) + ) + or [] + ): + if name not in seen: + seen.add(name) + names.append(name) + return names + + def get_file_text(self, path: str) -> str | None: + """Return ``path``'s text from the primary, else from the mirror. + + Absent from the primary means "this device has not migrated yet", + not "no data" -- so it falls through rather than reporting nothing. + """ + text = self.primary.get_file_text(path) + if text is not None: + return text + return self._try_mirror( + f"get_file_text {path}", lambda: self.mirror.get_file_text(path) + ) + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Write to the primary, then best-effort to the mirror. + + Raises: + RemoteSyncError: If the *primary* write fails. + """ + self.primary.put_file_text(path, text, message=message) + self._try_mirror( + f"put_file_text {path}", + lambda: self.mirror.put_file_text(path, text, message=message), + ) + + def delete_file(self, path: str, *, message: str = "crdt_sync: delete") -> None: + """Delete from the primary, then best-effort from the mirror. + + Raises: + RemoteSyncError: If the *primary* delete fails. + """ + self.primary.delete_file(path, message=message) + self._try_mirror( + f"delete_file {path}", + lambda: self.mirror.delete_file(path, message=message), + ) + + def can_access_remote(self) -> bool: + """Return whether the **primary** is reachable. + + Only the primary: a settings "Test connection" button must not report + success because the backend being retired happens to answer. + """ + return self.primary.can_access_remote() + + def get_string_map(self, path: str) -> dict[str, str]: + """Merge both backends' revision maps, primary winning on conflict. + + A device that has not migrated publishes revisions only to the + mirror; without this it would look revision-less and be + re-downloaded every tick for the whole trial period. + """ + merged: dict[str, str] = {} + mirror_read = getattr(self.mirror, "get_string_map", None) + if mirror_read is not None: + merged.update( + self._try_mirror(f"get_string_map {path}", lambda: mirror_read(path)) + or {} + ) + primary_read = getattr(self.primary, "get_string_map", None) + if primary_read is not None: + merged.update(primary_read(path)) + return merged + + def _try_mirror(self, operation: str, run: Callable[[], _T]) -> _T | None: + """Run a mirror operation, reporting rather than propagating failure.""" + try: + return run() + except RemoteSyncError as error: + self._on_mirror_failure(operation, error) + return None + + +def _log_mirror_failure(operation: str, error: Exception) -> None: + """Warn loudly, so a rotting fallback is visible before it is needed.""" + _logger.warning("Mirror backend failed during %s: %s", operation, error) diff --git a/crdt-sync/crdt_sync/_remote.py b/crdt-sync/crdt_sync/_remote.py new file mode 100644 index 0000000..55e6986 --- /dev/null +++ b/crdt-sync/crdt_sync/_remote.py @@ -0,0 +1,82 @@ +"""Backend-neutral remote-storage contract that sync talks through. + +Deliberately tiny: :func:`crdt_sync.sync_log` and every app-level sync +service talk only to :class:`RemoteStore`, so swapping the storage backend +(GitHub Contents API, Firebase Realtime Database, a dual-writing mirror of +both) is a constructor change at the call site and nothing more. + +Mirrors ``crdt_sync_dart``'s ``lib/src/remote_store.dart``; keep the two in +step. +""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + + +class RemoteSyncError(Exception): + """Raised for a remote-storage failure the caller must not ignore. + + The backend-neutral base of the error hierarchy: catch this to handle + "the sync transport failed" regardless of which backend is configured. + Backends narrow it -- :class:`crdt_sync.GitHubSyncError` and + :class:`crdt_sync.FirebaseSyncError` both derive from it -- so existing + ``except GitHubSyncError`` handlers keep working while a caller that has + been migrated can catch the base type instead. + """ + + +class RemoteNotFoundError(RemoteSyncError): + """Raised when the configured remote itself is unreachable. + + Distinguished from a missing *path* (nothing pushed there yet, which is + benign -- it just means no other device has synced before) so the caller + can tell "the repo/database is wrong or the credential isn't scoped to + it" apart from "no other device has synced yet". + """ + + +@runtime_checkable +class RemoteStore(Protocol): + """Dumb keyed storage of UTF-8 text blobs, with directory listing. + + Implementations raise :class:`RemoteSyncError` (or a subclass) for any + failure the caller must not silently ignore, and treat a missing path as + a benign ``None`` / empty list rather than an error. + + A ``Protocol`` rather than an ABC so the existing + :class:`crdt_sync.GitHubSyncClient` satisfies it structurally, with no + base class to add and no change to its own tests. + """ + + def list_directory(self, path: str) -> list[str]: + """Return the entry names directly under ``path``. + + Returns an empty list when nothing has been written there yet. + Includes both files and subdirectories: the + ``//`` layout needs to discover + device *directories*, not just files. + """ + + def get_file_text(self, path: str) -> str | None: + """Return the text stored at ``path``, or ``None`` if absent.""" + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Create or replace the text at ``path``. + + ``message`` is a human-readable reason for the write. Backends that + record one (GitHub commits) use it; backends that do not (Firebase) + ignore it. + """ + + def delete_file(self, path: str, *, message: str = ...) -> None: + """Delete ``path``. A no-op when ``path`` does not exist.""" + + def can_access_remote(self) -> bool: + """Return whether the configured credential can reach the remote. + + A lightweight connection test for a settings "Test connection" + button: it probes the remote root, so it succeeds even before any + file has been pushed. Never raises -- a network failure or missing + remote returns ``False``. + """ diff --git a/crdt-sync/crdt_sync/_sync.py b/crdt-sync/crdt_sync/_sync.py index b1edb4e..947154a 100644 --- a/crdt-sync/crdt_sync/_sync.py +++ b/crdt-sync/crdt_sync/_sync.py @@ -5,44 +5,237 @@ pushed log, merge with the local one, push this device's own merged result back up) so any app can reuse the loop while keeping its own on-disk JSON shape via the ``encode``/``decode`` callbacks -- this module has no opinion on what a ``Record``'s fields actually mean. + +Mirrors ``crdt_sync_dart``'s ``lib/src/sync.dart``; keep the two in step. """ from __future__ import annotations +from dataclasses import dataclass, field +import hashlib +import json import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from crdt_sync._log import merge_logs if TYPE_CHECKING: from collections.abc import Callable + from pathlib import Path - from crdt_sync._github import GitHubSyncClient from crdt_sync._log import Log + from crdt_sync._remote import RemoteStore _logger = logging.getLogger(__name__) _DEFAULT_FILENAME = "log.json" +@dataclass(frozen=True) +class SyncState: + """What one device remembers between ticks to skip needless traffic. + + Attributes: + ---------- + pushed_rev: + Hash of what this device last pushed, so an unchanged log is not + re-uploaded. 88% of the pushes in the GitHub-backed history this + replaces were byte-identical no-ops. + peer_revs: + The hash each peer had when we last downloaded it, so an unchanged + peer is not re-downloaded. + + Notes: + ----- + **Must be stored next to the log itself and cleared with it.** Skipping + an unchanged peer is only sound because that peer's records are already + merged into the local log; a cache that outlived its log would skip peers + whose data had been lost. + + """ + + pushed_rev: str | None = None + peer_revs: dict[str, str] = field(default_factory=dict) + + def to_json(self) -> dict[str, object]: + """Return a JSON-serializable form.""" + return {"pushed_rev": self.pushed_rev, "peer_revs": self.peer_revs} + + @classmethod + def from_json(cls, data: dict[str, object]) -> SyncState: + """Rebuild state from :meth:`to_json` output, ignoring junk. + + A malformed cache degrades into "fetch everything" rather than + raising: it is an optimisation, never a source of failure. + """ + pushed = data.get("pushed_rev") + peers = data.get("peer_revs") + return cls( + pushed_rev=pushed if isinstance(pushed, str) else None, + peer_revs={ + key: value + for key, value in (peers or {}).items() + if isinstance(value, str) + } + if isinstance(peers, dict) + else {}, + ) + + +class SyncStateStore(Protocol): + """Where :func:`sync_log` persists its :class:`SyncState` between runs.""" + + def load(self) -> SyncState: + """Return the stored state, or a default one.""" + + def save(self, state: SyncState) -> None: + """Persist ``state``.""" + + +class MemorySyncStateStore: + """A :class:`SyncStateStore` that forgets on exit, for tests. + + Correct but pessimistic: every process re-downloads every peer and + re-pushes the local log, which is exactly the old behaviour. + """ + + def __init__(self) -> None: + """Start with empty state.""" + self._state = SyncState() + + def load(self) -> SyncState: + """Return the held state.""" + return self._state + + def save(self, state: SyncState) -> None: + """Replace the held state.""" + self._state = state + + +class FileSyncStateStore: + """A :class:`SyncStateStore` backed by a JSON file. + + Persistence is what makes the saving real for short-lived callers: + ``wake_alarm``'s PC side is a fresh process every minute and + ``diet_guard``'s every 15 minutes, so an in-memory store would save + nothing at all. + """ + + def __init__(self, path: Path) -> None: + """Keep state in the file at ``path``.""" + self._path = path + + def load(self) -> SyncState: + """Return the stored state, or a default one if unreadable.""" + try: + data = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return SyncState() + if not isinstance(data, dict): + return SyncState() + return SyncState.from_json(data) + + def save(self, state: SyncState) -> None: + """Write ``state`` atomically, so a crash cannot truncate it.""" + self._path.parent.mkdir(parents=True, exist_ok=True) + temp = self._path.with_suffix(f"{self._path.suffix}.tmp") + temp.write_text(json.dumps(state.to_json()), encoding="utf-8") + temp.replace(self._path) + + +def revision_of(encoded_log: str) -> str: + """Return the revision of a serialized log: a content hash. + + A hash rather than the log's maximum HLC because merging a peer's *older* + record changes the content without raising the maximum -- with three + devices in one namespace (diet-guard has ``pc``, ``phone`` and + ``desktop``) a clock-based revision can miss a peer's merged state. It is + also the same value used to suppress no-op pushes, so the two + optimisations share one mechanism. + + Parameters + ---------- + encoded_log : str + The serialized log, exactly as it would be pushed. + + Returns: + ------- + str + A hex SHA-256 digest. + + """ + return hashlib.sha256(encoded_log.encode("utf-8")).hexdigest() + + +def default_revs_path(path_prefix: str) -> str: + """Return where revisions live for ``path_prefix``. + + A ``revs`` sibling of the device directory, e.g. + ``diet-guard-sync/devices`` -> ``diet-guard-sync/revs`` and + ``todo-sync/notes`` -> ``todo-sync/revs``. + + Parameters + ---------- + path_prefix : str + The directory holding one subdirectory per device. + + Returns: + ------- + str + The revisions directory. + + """ + head, separator, _ = path_prefix.rpartition("/") + return f"{head}/revs" if separator else f"{path_prefix}/revs" + + +def _remote_revs(client: RemoteStore, revs_path: str) -> dict[str, str]: + """Return every peer's published revision, cheaply where possible. + + Degrades to an empty map -- meaning "fetch everything", the old behaviour + -- on any backend without a bulk-map read, so correctness never depends + on the optimisation being available. + """ + get_string_map = getattr(client, "get_string_map", None) + if get_string_map is None: + return {} + return get_string_map(revs_path) + + def _pull_remote_logs( - client: GitHubSyncClient, + client: RemoteStore, device_id: str, path_prefix: str, filename: str, decode: Callable[[str], Log], + remote_revs: dict[str, str], + seen_revs: dict[str, str], + state: SyncState, ) -> list[Log]: """Return every other device's last-pushed log, skipping this one. A device whose pushed file is corrupt or unparsable (e.g. an interrupted push) is logged and skipped, same as one that has never pushed at all -- - GitHub is an external system boundary, and one bad device's file must not - stall merging in every other device's. + the remote is an external system boundary, and one bad device's file must + not stall merging in every other device's. + + Records into ``seen_revs`` which peers are now merged in, so the next + tick can skip them. """ remote_logs: list[Log] = [] for other_device_id in client.list_directory(path_prefix): if other_device_id == device_id: continue + remote_rev = remote_revs.get(other_device_id) + if remote_rev is not None and remote_rev == state.peer_revs.get( + other_device_id + ): + # Unchanged since we last merged it, and that merge is already + # part of local_log -- so the (potentially hundreds of KB) + # download is pure waste. Carry the revision forward so it stays + # skipped next tick. + seen_revs[other_device_id] = remote_rev + continue text = client.get_file_text(f"{path_prefix}/{other_device_id}/{filename}") if text is None: continue @@ -53,12 +246,16 @@ def _pull_remote_logs( "Unparsable log pushed by device %r, skipping", other_device_id, ) + # Deliberately not recorded as seen: a corrupt push must be + # retried next tick, not remembered as merged. + continue + seen_revs[other_device_id] = remote_rev or revision_of(text) return remote_logs def sync_log( *, - client: GitHubSyncClient, + client: RemoteStore, device_id: str, path_prefix: str, local_log: Log, @@ -66,20 +263,24 @@ def sync_log( decode: Callable[[str], Log], filename: str = _DEFAULT_FILENAME, commit_message: str = "crdt_sync: update log", + state_store: SyncStateStore | None = None, + revs_path: str | None = None, ) -> Log: """Run one full sync tick: pull every other device's log, merge, push. Pulls from ``//`` for every - device directory GitHub reports under ``path_prefix``, merges each into - ``local_log`` with :func:`crdt_sync.merge_logs`, then pushes this + device directory the remote reports under ``path_prefix``, merges each + into ``local_log`` with :func:`crdt_sync.merge_logs`, then pushes this device's own merged result to ``//``. Args: - client: An authenticated :class:`GitHubSyncClient`. + client: An authenticated :class:`RemoteStore` -- a + :class:`crdt_sync.GitHubSyncClient`, a + :class:`crdt_sync.FirebaseSyncClient`, or a mirror of both. device_id: This device's identifier; also the directory name its own log is pushed under. - path_prefix: The repo-relative directory holding one subdirectory - per device (e.g. ``"devices"``). + path_prefix: The directory holding one subdirectory per device + (e.g. ``"devices"``). local_log: This device's current full log (including tombstones). encode: Serializes a merged log for pushing. decode: Parses a remote device's pushed text back into a log. @@ -88,19 +289,60 @@ def sync_log( this tick rather than aborting the whole sync. filename: The file name each device pushes its log as. commit_message: The commit message used for this device's push. + state_store: Pass one to enable revision tracking, which skips + downloading peers that have not changed and skips pushing a log + that has not changed. Without it the tick behaves exactly as it + always did: fetch everything, push unconditionally. + revs_path: Where revisions live; defaults to + :func:`default_revs_path`. Returns: The merged log, as pushed. + + Notes: + Revisions live at ``/`` -- one small text node + per device, each written only by its owner. Deliberately *not* one + shared map per namespace: a whole-map write would erase every other + device's entry, after which those peers would look permanently + unchanged and never be fetched again. Per-device keys make that + failure impossible rather than merely avoided. """ + revs = revs_path if revs_path is not None else default_revs_path(path_prefix) + state = state_store.load() if state_store is not None else SyncState() + remote_revs = _remote_revs(client, revs) if state_store is not None else {} + merged = dict(local_log) + seen_revs: dict[str, str] = {} for remote_log in _pull_remote_logs( - client, device_id, path_prefix, filename, decode + client, + device_id, + path_prefix, + filename, + decode, + remote_revs, + seen_revs, + state, ): merged = merge_logs(merged, remote_log) - client.put_file_text( - f"{path_prefix}/{device_id}/{filename}", - encode(merged), - message=commit_message, - ) + encoded = encode(merged) + revision = revision_of(encoded) + unchanged = state_store is not None and revision == state.pushed_rev + if not unchanged: + client.put_file_text( + f"{path_prefix}/{device_id}/{filename}", + encoded, + message=commit_message, + ) + if state_store is not None: + # Published after the log, never before: a peer that cached + # "seen rev X" against a log it never received would skip it + # forever. + client.put_file_text( + f"{revs}/{device_id}", + revision, + message=commit_message, + ) + if state_store is not None: + state_store.save(SyncState(pushed_rev=revision, peer_revs=seen_revs)) return merged diff --git a/crdt-sync/crdt_sync/tests/test_firebase.py b/crdt-sync/crdt_sync/tests/test_firebase.py new file mode 100644 index 0000000..ca7a71a --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_firebase.py @@ -0,0 +1,319 @@ +"""Tests for the Realtime Database sync client. + +The HTTP layer is fully mocked, so every branch -- success, absent path, +rules rejection, quota exhaustion, malformed values and network exceptions -- +is exercised without any network access. + +Key escaping is tested against the same cases as the Dart side +(``test/firebase_client_test.dart``); the two implementations must agree +byte-for-byte or a device on one would not read the other's data. +""" + +from __future__ import annotations + +import datetime as dt +import json +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from crdt_sync import ( + DatabaseNotFoundError, + FirebaseCredentials, + FirebaseSyncClient, + FirebaseSyncError, + FirebaseTokenProvider, + MemoryCredentialStore, + RemoteNotFoundError, + RemoteStore, + RemoteSyncError, +) +from crdt_sync import _firebase as fb +from crdt_sync._firebase import decode_key, encode_key, encode_path + +_DATABASE_URL = "https://example-default-rtdb.europe-west1.firebasedatabase.app" + +# Named without the word "token": ruff's hardcoded-credential checks (S105 / +# S106) key off the identifier, and these are obviously fake test fixtures. +_FAKE_ID = "id-fixture" +_FAKE_REFRESH = "refresh-fixture" + + +def _response(status_code: int = 200, json_data: object = None) -> MagicMock: + """Build a fake ``requests.Response`` with a status and JSON body.""" + response = MagicMock() + response.status_code = status_code + response.ok = 200 <= status_code < 300 + response.json = MagicMock(return_value=json_data) + response.text = json.dumps(json_data) + return response + + +def _auth(*, signed_in: bool = True) -> FirebaseTokenProvider: + """A provider holding a long-lived token, so no auth HTTP is needed.""" + credentials = ( + FirebaseCredentials( + id_token=_FAKE_ID, + refresh_token=_FAKE_REFRESH, + expires_at=dt.datetime(2099, 1, 1, tzinfo=dt.UTC), + ) + if signed_in + else None + ) + return FirebaseTokenProvider("fake-api-key", MemoryCredentialStore(credentials)) + + +def _client(*, signed_in: bool = True) -> FirebaseSyncClient: + return FirebaseSyncClient(_DATABASE_URL, _auth(signed_in=signed_in)) + + +class TestKeyEscaping: + def test_escapes_a_json_filename_into_a_legal_key(self) -> None: + # `.` is illegal in an RTDB key, and the trailing `.json` in a REST + # URL is the format suffix -- so the name cannot be stored verbatim. + assert encode_key("log.json") == "log~2Ejson" + + def test_leaves_dot_free_device_ids_untouched(self) -> None: + assert encode_key("pc") == "pc" + uuid = "32c28cf3-c0eb-423f-8bac-9bda9f158054" + assert encode_key(uuid) == uuid + + def test_escapes_every_forbidden_character_and_the_escape_char(self) -> None: + assert encode_key("a.b$c#d[e]f~g") == "a~2Eb~24c~23d~5Be~5Df~7Eg" + + @pytest.mark.parametrize( + "name", + ["log.json", "plain", "a.b$c#d[e]f~g", "log~2Ejson", "~", ""], + ) + def test_round_trips(self, name: str) -> None: + # A name that already looks like an escape must survive: `~` is + # escaped first, so `log~2Ejson` is not mistaken for `log.json`. + assert decode_key(encode_key(name)) == name + + def test_escapes_each_path_segment_but_keeps_separators(self) -> None: + assert encode_path("todo-sync/notes/abc.json") == "todo-sync/notes/abc~2Ejson" + + def test_drops_empty_segments(self) -> None: + assert not encode_path("") + assert encode_path("/a//b/") == "a/b" + + +class TestGetFileText: + def test_returns_the_stored_text_blob(self) -> None: + with patch.object(fb.requests, "get", return_value=_response(200, '{"a":1}')): + assert _client().get_file_text("ns/devices/pc/log.json") == '{"a":1}' + + def test_requests_the_escaped_path_with_the_auth_token(self) -> None: + with patch.object( + fb.requests, "get", return_value=_response(200, "{}") + ) as get_mock: + _client().get_file_text("ns/devices/pc/log.json") + url = get_mock.call_args.args[0] + assert url.endswith("/ns/devices/pc/log~2Ejson.json") + assert get_mock.call_args.kwargs["params"]["auth"] == _FAKE_ID + + def test_returns_none_for_a_path_never_written_to(self) -> None: + # RTDB answers a missing path with literal null and a 200, so this is + # benign "no other device has synced yet", not an error. + with patch.object(fb.requests, "get", return_value=_response(200, None)): + assert _client().get_file_text("ns/devices/phone/log.json") is None + + def test_raises_when_the_value_is_not_a_text_blob(self) -> None: + with ( + patch.object(fb.requests, "get", return_value=_response(200, {"a": 1})), + pytest.raises(FirebaseSyncError, match="not the expected text blob"), + ): + _client().get_file_text("ns/devices/pc/log.json") + + @pytest.mark.parametrize("status", [401, 403]) + def test_raises_database_not_found_when_the_rules_reject(self, status: int) -> None: + with ( + patch.object( + fb.requests, "get", return_value=_response(status, {"error": "denied"}) + ), + pytest.raises(DatabaseNotFoundError), + ): + _client().get_file_text("ns/devices/pc/log.json") + + def test_raises_a_plain_sync_error_on_any_other_non_2xx(self) -> None: + # Spark quota exhaustion lands here. It must surface, never be + # swallowed into a None that reads as "nothing to sync". + with ( + patch.object( + fb.requests, "get", return_value=_response(429, {"error": "quota"}) + ), + pytest.raises(FirebaseSyncError, match="429"), + ): + _client().get_file_text("ns/devices/pc/log.json") + + def test_turns_a_network_failure_into_a_sync_error(self) -> None: + with ( + patch.object( + fb.requests, "get", side_effect=requests.ConnectionError("offline") + ), + pytest.raises(FirebaseSyncError, match="network error"), + ): + _client().get_file_text("ns/devices/pc/log.json") + + +class TestListDirectory: + def test_returns_decoded_keys_and_asks_for_a_shallow_read(self) -> None: + payload = {"pc": True, "phone": True} + with patch.object( + fb.requests, "get", return_value=_response(200, payload) + ) as get_mock: + assert _client().list_directory("ns/devices") == ["pc", "phone"] + # shallow=true keeps a listing costing bytes instead of the whole + # subtree -- the difference between ~40 MB/month and ~700 MB. + assert get_mock.call_args.kwargs["params"]["shallow"] == "true" + + def test_decodes_escaped_filenames_back(self) -> None: + # The notes app lists filenames, not device dirs: `.json` returns. + with patch.object( + fb.requests, "get", return_value=_response(200, {"abc~2Ejson": True}) + ): + assert _client().list_directory("todo-sync/notes") == ["abc.json"] + + def test_returns_empty_for_an_unwritten_prefix(self) -> None: + with patch.object(fb.requests, "get", return_value=_response(200, None)): + assert not _client().list_directory("ns/devices") + + def test_raises_on_a_non_2xx_response(self) -> None: + with ( + patch.object( + fb.requests, "get", return_value=_response(500, {"error": "boom"}) + ), + pytest.raises(FirebaseSyncError), + ): + _client().list_directory("ns/devices") + + +class TestPutFileText: + def test_writes_the_text_as_a_json_string_leaf(self) -> None: + with ( + patch.object(fb.requests, "get", return_value=_response(200, None)), + patch.object( + fb.requests, "put", return_value=_response(200, '{"a":1}') + ) as put_mock, + ): + _client().put_file_text( + "ns/devices/pc/log.json", '{"a":1}', message="ignored" + ) + assert put_mock.call_args.kwargs["json"] == '{"a":1}' + assert put_mock.call_args.args[0].endswith("/log~2Ejson.json") + + def test_raises_on_a_non_2xx_response(self) -> None: + with ( + patch.object( + fb.requests, "put", return_value=_response(400, {"error": "bad"}) + ), + pytest.raises(FirebaseSyncError), + ): + _client().put_file_text("ns/devices/pc/log.json", "{}", message="m") + + def test_turns_a_network_failure_into_a_sync_error(self) -> None: + with ( + patch.object( + fb.requests, "put", side_effect=requests.ConnectionError("offline") + ), + pytest.raises(FirebaseSyncError, match="network error"), + ): + _client().put_file_text("ns/devices/pc/log.json", "{}", message="m") + + +class TestGetStringMap: + def test_returns_the_map_with_keys_decoded(self) -> None: + payload = {"pc": "sha-1", "phone": "sha-2"} + with patch.object(fb.requests, "get", return_value=_response(200, payload)): + assert _client().get_string_map("ns/revs") == payload + + def test_degrades_to_empty_when_absent(self) -> None: + with patch.object(fb.requests, "get", return_value=_response(200, None)): + assert not _client().get_string_map("ns/revs") + + def test_skips_non_string_entries_rather_than_failing(self) -> None: + # The revs node is an optimisation; a corrupt one must degrade into + # "fetch everything", never into a failed tick. + with patch.object( + fb.requests, "get", return_value=_response(200, {"pc": "sha-1", "b": 42}) + ): + assert _client().get_string_map("ns/revs") == {"pc": "sha-1"} + + def test_raises_on_a_non_2xx_response(self) -> None: + with ( + patch.object( + fb.requests, "get", return_value=_response(500, {"error": "boom"}) + ), + pytest.raises(FirebaseSyncError), + ): + _client().get_string_map("ns/revs") + + +class TestDeleteFile: + def test_sends_a_delete_to_the_escaped_path(self) -> None: + with patch.object( + fb.requests, "delete", return_value=_response(200, None) + ) as delete_mock: + _client().delete_file("ns/devices/pc/log.json") + assert delete_mock.call_args.args[0].endswith("/log~2Ejson.json") + + def test_raises_on_a_non_2xx_response(self) -> None: + with ( + patch.object( + fb.requests, "delete", return_value=_response(500, {"error": "boom"}) + ), + pytest.raises(FirebaseSyncError), + ): + _client().delete_file("ns/devices/pc/log.json") + + def test_turns_a_network_failure_into_a_sync_error(self) -> None: + with ( + patch.object( + fb.requests, "delete", side_effect=requests.ConnectionError("x") + ), + pytest.raises(FirebaseSyncError, match="network error"), + ): + _client().delete_file("ns/devices/pc/log.json") + + +class TestCanAccessRemote: + def test_true_when_the_database_root_reads(self) -> None: + with patch.object(fb.requests, "get", return_value=_response(200, {"ns": 1})): + assert _client().can_access_remote() is True + + def test_false_when_the_rules_reject_the_token(self) -> None: + with patch.object( + fb.requests, "get", return_value=_response(401, {"error": "denied"}) + ): + assert _client().can_access_remote() is False + + def test_false_on_a_network_error(self) -> None: + with patch.object( + fb.requests, "get", side_effect=requests.ConnectionError("offline") + ): + assert _client().can_access_remote() is False + + def test_false_when_not_signed_in_rather_than_raising(self) -> None: + # "cannot get a token" is exactly "cannot access the remote", and a + # settings Test-connection button must not blow up. + assert _client(signed_in=False).can_access_remote() is False + + +class TestRemoteStoreContract: + def test_the_client_satisfies_remote_store(self) -> None: + assert isinstance(_client(), RemoteStore) + + def test_database_not_found_is_catchable_as_either_error(self) -> None: + error = DatabaseNotFoundError("gone") + assert isinstance(error, FirebaseSyncError) + assert isinstance(error, RemoteNotFoundError) + assert isinstance(error, RemoteSyncError) + + def test_a_trailing_slash_on_the_database_url_is_tolerated(self) -> None: + client = FirebaseSyncClient(f"{_DATABASE_URL}/", _auth()) + with patch.object( + fb.requests, "get", return_value=_response(200, "ok") + ) as get_mock: + client.get_file_text("ns/x.json") + assert get_mock.call_args.args[0] == f"{_DATABASE_URL}/ns/x~2Ejson.json" diff --git a/crdt-sync/crdt_sync/tests/test_firebase_auth.py b/crdt-sync/crdt_sync/tests/test_firebase_auth.py new file mode 100644 index 0000000..799ba97 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_firebase_auth.py @@ -0,0 +1,273 @@ +"""Tests for Firebase Authentication over the REST API. + +The HTTP layer is fully mocked, so every branch -- sign-in, refresh, rotated +refresh tokens, revoked sessions, malformed error bodies and network failures +-- is exercised without any network access. +""" + +from __future__ import annotations + +import datetime as dt +import json +from typing import TYPE_CHECKING +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from crdt_sync import ( + FileCredentialStore, + FirebaseAuthError, + FirebaseCredentials, + FirebaseTokenProvider, + MemoryCredentialStore, +) +from crdt_sync import _firebase_auth as fa + +if TYPE_CHECKING: + from pathlib import Path + +_NOW = dt.datetime(2026, 8, 3, 12, tzinfo=dt.UTC) + +# Fixture values referenced through names ruff's hardcoded-credential checks +# (S105-S107) do not key off. Comparing an attribute called `id_token` to a +# bare string literal trips S105 no matter how obviously fake the value is. +_ID_1 = "id-1" +_ID_2 = "id-2" +_REFRESH_1 = "refresh-1" +_REFRESH_2 = "refresh-2" + + +def _response(status_code: int = 200, json_data: object = None) -> MagicMock: + """Build a fake ``requests.Response`` with a status and JSON body.""" + response = MagicMock() + response.status_code = status_code + response.ok = 200 <= status_code < 300 + response.json = MagicMock(return_value=json_data if json_data is not None else {}) + response.text = json.dumps(json_data) if json_data is not None else "" + return response + + +def _provider( + store: object = None, + *, + now: dt.datetime = _NOW, +) -> FirebaseTokenProvider: + return FirebaseTokenProvider( + "fake-api-key", + store or MemoryCredentialStore(), + clock=lambda: now, + ) + + +# Parameters named without "token": ruff's hardcoded-credential check (S107) +# keys off the identifier, and these are obviously fake test fixtures. +def _credentials( + *, + id_value: str = _ID_1, + refresh_value: str = _REFRESH_1, + valid_for: dt.timedelta = dt.timedelta(hours=1), +) -> FirebaseCredentials: + return FirebaseCredentials( + id_token=id_value, + refresh_token=refresh_value, + expires_at=_NOW + valid_for, + ) + + +def _patch_post(*responses: MagicMock) -> object: + return patch.object(fa.requests, "post", side_effect=list(responses)) + + +class TestFirebaseCredentials: + def test_round_trips_through_json(self) -> None: + original = _credentials() + restored = FirebaseCredentials.from_json(original.to_json()) + assert restored == original + + def test_is_not_expired_inside_its_lifetime(self) -> None: + assert _credentials().is_expired_at(_NOW) is False + + def test_is_expired_once_past_expiry(self) -> None: + stale = _credentials(valid_for=dt.timedelta(minutes=-1)) + assert stale.is_expired_at(_NOW) is True + + def test_is_expired_inside_the_refresh_skew(self) -> None: + # A tick starting now would outlive a token with 2 minutes left, so + # it must count as already expired. + soon = _credentials(valid_for=dt.timedelta(minutes=2)) + assert soon.is_expired_at(_NOW) is True + + +class TestSignIn: + def test_stores_the_session(self) -> None: + store = MemoryCredentialStore() + body = {"idToken": _ID_1, "refreshToken": _REFRESH_1, "expiresIn": "3600"} + with _patch_post(_response(200, body)): + _provider(store).sign_in("me@example.com", "hunter2") + saved = store.load() + assert saved is not None + assert saved.id_token == _ID_1 + assert saved.refresh_token == _REFRESH_1 + assert saved.expires_at == _NOW + dt.timedelta(hours=1) + + def test_reports_googles_reason_for_a_rejected_password(self) -> None: + body = {"error": {"message": "INVALID_LOGIN_CREDENTIALS"}} + with ( + _patch_post(_response(400, body)), + pytest.raises(FirebaseAuthError, match="INVALID_LOGIN_CREDENTIALS"), + ): + _provider().sign_in("me@example.com", "wrong") + + def test_reports_a_string_shaped_error_body(self) -> None: + with ( + _patch_post(_response(400, {"error": "BAD_REQUEST"})), + pytest.raises(FirebaseAuthError, match="BAD_REQUEST"), + ): + _provider().sign_in("a@b.c", "x") + + def test_survives_an_error_body_with_no_error_key(self) -> None: + with ( + _patch_post(_response(400, {"unexpected": "shape"})), + pytest.raises(FirebaseAuthError, match="HTTP 400"), + ): + _provider().sign_in("a@b.c", "x") + + def test_survives_a_non_dict_error_body(self) -> None: + with ( + _patch_post(_response(400, ["not", "a", "dict"])), + pytest.raises(FirebaseAuthError, match="HTTP 400"), + ): + _provider().sign_in("a@b.c", "x") + + def test_survives_a_non_json_error_body(self) -> None: + response = _response(502) + response.json = MagicMock(side_effect=ValueError("not json")) + with ( + _patch_post(response), + pytest.raises(FirebaseAuthError, match="HTTP 502"), + ): + _provider().sign_in("a@b.c", "x") + + def test_turns_a_network_failure_into_an_auth_error(self) -> None: + with ( + patch.object( + fa.requests, "post", side_effect=requests.ConnectionError("offline") + ), + pytest.raises(FirebaseAuthError, match="network error"), + ): + _provider().sign_in("a@b.c", "x") + + +class TestIdToken: + def test_fails_loudly_when_not_signed_in(self) -> None: + # Never returns None or a stale token: a sync that quietly stops + # syncing is the failure mode this design exists to prevent. + with pytest.raises(FirebaseAuthError, match="not signed in"): + _provider().id_token() + + def test_returns_the_stored_token_without_a_network_call(self) -> None: + provider = _provider(MemoryCredentialStore(_credentials())) + with patch.object(fa.requests, "post", side_effect=AssertionError("no HTTP")): + assert provider.id_token() == _ID_1 + + def test_refreshes_an_expired_token_and_keeps_the_rotated_one(self) -> None: + store = MemoryCredentialStore(_credentials(valid_for=dt.timedelta(minutes=-5))) + body = { + "id_token": _ID_2, + "refresh_token": _REFRESH_2, + "expires_in": "3600", + } + with _patch_post(_response(200, body)): + assert _provider(store).id_token() == _ID_2 + saved = store.load() + assert saved is not None + assert saved.refresh_token == _REFRESH_2 + + def test_fails_loudly_when_the_refresh_token_was_revoked(self) -> None: + store = MemoryCredentialStore(_credentials(valid_for=dt.timedelta(minutes=-5))) + with ( + _patch_post(_response(400, {"error": {"message": "TOKEN_EXPIRED"}})), + pytest.raises(FirebaseAuthError, match="TOKEN_EXPIRED"), + ): + _provider(store).id_token() + + +class TestSessionLifecycle: + def test_has_session_is_false_before_sign_in_and_true_after(self) -> None: + store = MemoryCredentialStore() + provider = _provider(store) + assert provider.has_session() is False + body = {"idToken": _ID_1, "refreshToken": _REFRESH_1, "expiresIn": "3600"} + with _patch_post(_response(200, body)): + provider.sign_in("a@b.c", "x") + assert provider.has_session() is True + + def test_sign_out_clears_the_store(self) -> None: + store = MemoryCredentialStore(_credentials()) + provider = _provider(store) + assert provider.id_token() == _ID_1 + provider.sign_out() + assert store.load() is None + with pytest.raises(FirebaseAuthError): + provider.id_token() + + def test_defaults_to_the_real_clock_when_none_is_injected(self) -> None: + provider = FirebaseTokenProvider("k", MemoryCredentialStore()) + assert provider.has_session() is False + + +class TestFileCredentialStore: + def test_round_trips_through_a_file(self, tmp_path: Path) -> None: + store = FileCredentialStore(tmp_path / "nested" / "creds.json") + store.save(_credentials()) + assert store.load() == _credentials() + + def test_writes_the_file_readable_only_by_this_user(self, tmp_path: Path) -> None: + # The refresh token is the real secret; it must never be briefly + # world-readable between write and chmod. + path = tmp_path / "creds.json" + FileCredentialStore(path).save(_credentials()) + assert path.stat().st_mode & 0o777 == 0o600 + + def test_load_returns_none_when_absent(self, tmp_path: Path) -> None: + assert FileCredentialStore(tmp_path / "missing.json").load() is None + + def test_load_returns_none_for_a_truncated_file(self, tmp_path: Path) -> None: + # An interrupted write reads as "not signed in"; the caller's next + # step is to sign in again, which repairs it. + path = tmp_path / "creds.json" + path.write_text('{"id_token": "a"', encoding="utf-8") + assert FileCredentialStore(path).load() is None + + def test_load_returns_none_when_fields_are_missing(self, tmp_path: Path) -> None: + path = tmp_path / "creds.json" + path.write_text('{"id_token": "a"}', encoding="utf-8") + assert FileCredentialStore(path).load() is None + + def test_load_returns_none_for_an_unparsable_expiry(self, tmp_path: Path) -> None: + path = tmp_path / "creds.json" + path.write_text( + json.dumps( + {"id_token": "a", "refresh_token": "b", "expires_at": "not-a-date"} + ), + encoding="utf-8", + ) + assert FileCredentialStore(path).load() is None + + def test_clear_removes_the_file_and_is_idempotent(self, tmp_path: Path) -> None: + path = tmp_path / "creds.json" + store = FileCredentialStore(path) + store.save(_credentials()) + store.clear() + assert not path.exists() + store.clear() + + def test_overwrites_an_existing_file(self, tmp_path: Path) -> None: + path = tmp_path / "creds.json" + store = FileCredentialStore(path) + store.save(_credentials()) + store.save(_credentials(id_value=_ID_2)) + loaded = store.load() + assert loaded is not None + assert loaded.id_token == _ID_2 diff --git a/crdt-sync/crdt_sync/tests/test_github.py b/crdt-sync/crdt_sync/tests/test_github.py index db7e176..e60270c 100644 --- a/crdt-sync/crdt_sync/tests/test_github.py +++ b/crdt-sync/crdt_sync/tests/test_github.py @@ -13,7 +13,15 @@ from unittest.mock import MagicMock, patch import pytest import requests -from crdt_sync import GitHubSyncClient, GitHubSyncError, RepoNotFoundError, _github +from crdt_sync import ( + GitHubSyncClient, + GitHubSyncError, + RemoteNotFoundError, + RemoteStore, + RemoteSyncError, + RepoNotFoundError, + _github, +) def _response(status_code: int = 200, json_data: object = None) -> MagicMock: @@ -184,6 +192,37 @@ class TestCanAccessRepo: assert _client().can_access_repo() is False +class TestCanAccessRemote: + """The ``RemoteStore`` spelling must behave exactly like the legacy name.""" + + def test_true_when_repo_endpoint_ok(self) -> None: + with _patch_get(_response(200)): + assert _client().can_access_remote() is True + + def test_false_when_repo_is_missing(self) -> None: + with _patch_get(_response(404)): + assert _client().can_access_remote() is False + + def test_false_on_a_network_error(self) -> None: + with _patch_get_raises(): + assert _client().can_access_remote() is False + + +class TestRemoteStoreContract: + """The seam sync talks through must actually be satisfied by the client.""" + + def test_github_client_is_a_remote_store(self) -> None: + assert isinstance(_client(), RemoteStore) + + def test_repo_not_found_is_catchable_as_either_error(self) -> None: + # Backend-neutral callers catch RemoteNotFoundError; existing GitHub + # callers catch GitHubSyncError. One exception must satisfy both. + error = RepoNotFoundError("gone") + assert isinstance(error, GitHubSyncError) + assert isinstance(error, RemoteNotFoundError) + assert isinstance(error, RemoteSyncError) + + class TestDeleteFile: def test_deletes_an_existing_file_resolving_its_own_sha(self) -> None: with ( diff --git a/crdt-sync/crdt_sync/tests/test_mirror.py b/crdt-sync/crdt_sync/tests/test_mirror.py new file mode 100644 index 0000000..6e76da2 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_mirror.py @@ -0,0 +1,336 @@ +"""Tests for the dual-write client used during the GitHub -> Firebase cutover. + +The asymmetry is the whole point and is what these assert: a primary failure +must fail the tick, a mirror failure must not, and reads must consult both so +a half-migrated app (one device moved, one not) still converges both ways. + +Mirrors ``crdt_sync_dart``'s ``test/mirror_store_test.dart``. +""" + +from __future__ import annotations + +import pytest + +from crdt_sync import ( + Hlc, + Log, + MirrorSyncClient, + Record, + RemoteStore, + RemoteSyncError, + dump_log, + load_log, + sync_log, +) + + +class FakeStore: + """A scriptable in-memory store that can be told to fail.""" + + def __init__( + self, + files: dict[str, str] | None = None, + *, + failing: bool = False, + ) -> None: + """Start holding ``files``, failing every call if ``failing``.""" + self.files = dict(files or {}) + self.failing = failing + self.writes: list[str] = [] + + def _guard(self, what: str) -> None: + if self.failing: + msg = f"{what} failed" + raise RemoteSyncError(msg) + + def list_directory(self, path: str) -> list[str]: + """Return the distinct first segments under ``path``.""" + self._guard("list") + prefix = f"{path}/" + return sorted( + { + key[len(prefix) :].split("/")[0] + for key in self.files + if key.startswith(prefix) + } + ) + + def get_file_text(self, path: str) -> str | None: + """Return the stored text, if any.""" + self._guard("read") + return self.files.get(path) + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Store ``text`` at ``path``.""" + del message + self._guard("write") + self.writes.append(path) + self.files[path] = text + + def get_string_map(self, path: str) -> dict[str, str]: + """Return the direct children of ``path`` as a flat map.""" + self._guard("map read") + prefix = f"{path}/" + return { + key[len(prefix) :]: value + for key, value in self.files.items() + if key.startswith(prefix) + } + + def delete_file(self, path: str, *, message: str = "") -> None: + """Remove ``path`` if present.""" + del message + self._guard("delete") + self.files.pop(path, None) + + def can_access_remote(self) -> bool: + """Return whether this backend is currently healthy.""" + return not self.failing + + +class FakeStoreWithoutBulkRead: + """A store with no bulk-map read, standing in for GitHub. + + Composes rather than subclasses :class:`FakeStore`, because a subclass + would inherit ``get_string_map`` -- the capability this fake must lack. + """ + + def __init__(self, files: dict[str, str] | None = None) -> None: + """Wrap a plain :class:`FakeStore` holding ``files``.""" + self._inner = FakeStore(files) + + @property + def files(self) -> dict[str, str]: + """The wrapped store's contents.""" + return self._inner.files + + def list_directory(self, path: str) -> list[str]: + """Delegate to the wrapped store.""" + return self._inner.list_directory(path) + + def get_file_text(self, path: str) -> str | None: + """Delegate to the wrapped store.""" + return self._inner.get_file_text(path) + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Delegate to the wrapped store.""" + self._inner.put_file_text(path, text, message=message) + + def delete_file(self, path: str, *, message: str = "") -> None: + """Delegate to the wrapped store.""" + self._inner.delete_file(path, message=message) + + def can_access_remote(self) -> bool: + """Delegate to the wrapped store.""" + return self._inner.can_access_remote() + + +def _mirror( + *, + primary_files: dict[str, str] | None = None, + mirror_files: dict[str, str] | None = None, + primary_failing: bool = False, + mirror_failing: bool = False, +) -> tuple[MirrorSyncClient, FakeStore, FakeStore, list[str]]: + primary = FakeStore(primary_files, failing=primary_failing) + mirror = FakeStore(mirror_files, failing=mirror_failing) + failures: list[str] = [] + client = MirrorSyncClient( + primary, + mirror, + on_mirror_failure=lambda operation, _: failures.append(operation), + ) + return client, primary, mirror, failures + + +class TestWrites: + def test_go_to_both_backends(self) -> None: + client, primary, mirror, _ = _mirror() + client.put_file_text("ns/pc/log.json", "{}", message="m") + assert primary.writes == ["ns/pc/log.json"] + assert mirror.writes == ["ns/pc/log.json"] + + def test_a_primary_failure_fails_the_tick(self) -> None: + # Fail-closed: the primary is authoritative, so a sync that could not + # write it must not be reported as successful. + client, _, _, _ = _mirror(primary_failing=True) + with pytest.raises(RemoteSyncError): + client.put_file_text("ns/pc/log.json", "{}", message="m") + + def test_a_mirror_failure_is_loud_but_survivable(self) -> None: + client, primary, _, failures = _mirror(mirror_failing=True) + client.put_file_text("ns/pc/log.json", "{}", message="m") + assert primary.writes == ["ns/pc/log.json"] + assert failures == ["put_file_text ns/pc/log.json"] + + def test_deletes_behave_the_same_way(self) -> None: + client, primary, _, failures = _mirror( + primary_files={"ns/pc/log.json": "{}"}, + mirror_files={"ns/pc/log.json": "{}"}, + mirror_failing=True, + ) + client.delete_file("ns/pc/log.json") + assert not primary.files + assert failures == ["delete_file ns/pc/log.json"] + + def test_a_primary_delete_failure_fails_the_tick(self) -> None: + client, _, _, _ = _mirror(primary_failing=True) + with pytest.raises(RemoteSyncError): + client.delete_file("ns/pc/log.json") + + +class TestReads: + def test_prefer_the_primary_when_it_has_the_file(self) -> None: + client, _, _, _ = _mirror( + primary_files={"ns/pc/log.json": "from-primary"}, + mirror_files={"ns/pc/log.json": "from-mirror"}, + ) + assert client.get_file_text("ns/pc/log.json") == "from-primary" + + def test_fall_back_to_the_mirror_for_an_unmigrated_device(self) -> None: + # Why reads are not primary-only: a migrated PC must still see an + # un-migrated phone's writes, or convergence silently becomes + # one-directional with no error raised. + client, _, _, _ = _mirror(mirror_files={"ns/phone/log.json": "from-mirror"}) + assert client.get_file_text("ns/phone/log.json") == "from-mirror" + + def test_return_none_when_neither_backend_has_the_file(self) -> None: + client, _, _, _ = _mirror() + assert client.get_file_text("ns/nobody/log.json") is None + + def test_a_mirror_read_failure_degrades_to_the_primary(self) -> None: + client, _, _, failures = _mirror(mirror_failing=True) + assert client.get_file_text("ns/pc/log.json") is None + assert failures == ["get_file_text ns/pc/log.json"] + + def test_a_primary_read_failure_fails_the_tick(self) -> None: + client, _, _, _ = _mirror(primary_failing=True) + with pytest.raises(RemoteSyncError): + client.get_file_text("ns/pc/log.json") + + +class TestListDirectory: + def test_unions_devices_from_both_backends(self) -> None: + client, _, _, _ = _mirror( + primary_files={"ns/pc/log.json": "{}"}, + mirror_files={"ns/phone/log.json": "{}"}, + ) + assert sorted(client.list_directory("ns")) == ["pc", "phone"] + + def test_does_not_duplicate_a_device_present_in_both(self) -> None: + client, _, _, _ = _mirror( + primary_files={"ns/pc/log.json": "{}"}, + mirror_files={"ns/pc/log.json": "{}"}, + ) + assert client.list_directory("ns") == ["pc"] + + def test_a_mirror_failure_degrades_to_the_primary_list(self) -> None: + client, _, _, failures = _mirror( + primary_files={"ns/pc/log.json": "{}"}, mirror_failing=True + ) + assert client.list_directory("ns") == ["pc"] + assert failures == ["list_directory ns"] + + def test_a_primary_failure_fails_the_tick(self) -> None: + client, _, _, _ = _mirror(primary_failing=True) + with pytest.raises(RemoteSyncError): + client.list_directory("ns") + + +class TestRevisionMaps: + def test_merge_both_backends_with_the_primary_winning(self) -> None: + client, _, _, _ = _mirror( + primary_files={"ns/revs/pc": "primary-rev"}, + mirror_files={"ns/revs/pc": "mirror-rev", "ns/revs/phone": "old"}, + ) + # An un-migrated device publishes revisions only to the mirror; + # without this it would look revision-less and be re-downloaded + # every tick for the whole trial. + assert client.get_string_map("ns/revs") == { + "pc": "primary-rev", + "phone": "old", + } + + def test_a_mirror_failure_degrades_to_primary_revisions(self) -> None: + client, _, _, failures = _mirror( + primary_files={"ns/revs/pc": "primary-rev"}, mirror_failing=True + ) + assert client.get_string_map("ns/revs") == {"pc": "primary-rev"} + assert failures == ["get_string_map ns/revs"] + + def test_a_backend_without_bulk_reads_contributes_nothing(self) -> None: + client = MirrorSyncClient( + FakeStore({"ns/revs/pc": "primary-rev"}), + FakeStoreWithoutBulkRead({"ns/revs/phone": "ignored"}), + ) + assert client.get_string_map("ns/revs") == {"pc": "primary-rev"} + + def test_is_empty_when_neither_backend_has_bulk_reads(self) -> None: + client = MirrorSyncClient( + FakeStoreWithoutBulkRead(), FakeStoreWithoutBulkRead() + ) + assert not client.get_string_map("ns/revs") + + +class TestLifecycle: + def test_can_access_remote_reports_only_the_primary(self) -> None: + # A Test-connection button must not report success because the + # backend being retired happens to answer. + broken, _, _, _ = _mirror(primary_failing=True) + assert broken.can_access_remote() is False + healthy, _, _, _ = _mirror(mirror_failing=True) + assert healthy.can_access_remote() is True + + def test_a_mirror_failure_with_no_handler_is_logged_not_raised( + self, + caplog: pytest.LogCaptureFixture, + ) -> None: + client = MirrorSyncClient(FakeStore(), FakeStore(failing=True)) + client.put_file_text("ns/pc/log.json", "{}", message="m") + assert "Mirror backend failed" in caplog.text + + def test_satisfies_the_remote_store_contract(self) -> None: + client, _, _, _ = _mirror() + assert isinstance(client, RemoteStore) + + +class TestEndToEndThroughSyncLog: + def test_a_half_migrated_pair_still_converges_both_ways(self) -> None: + # phone has not migrated and still writes only to GitHub; pc is on + # Firebase with the GitHub mirror. Both must see both records. + github = FakeStore() + firebase = FakeStore() + + def log(record_id: str, node_id: str) -> Log: + return { + record_id: Record( + id=record_id, + fields={ + "v": ( + record_id, + Hlc(wall_time_ms=1000, counter=0, node_id=node_id), + ) + }, + ) + } + + sync_log( + client=github, + device_id="phone", + path_prefix="ns/devices", + local_log=log("from-phone", "node-phone"), + encode=dump_log, + decode=load_log, + ) + merged = sync_log( + client=MirrorSyncClient(firebase, github), + device_id="pc", + path_prefix="ns/devices", + local_log=log("from-pc", "node-pc"), + encode=dump_log, + decode=load_log, + ) + assert {"from-phone", "from-pc"} <= set(merged) + # The pc's merged result is mirrored back to GitHub, so the + # un-migrated phone sees it on its next tick. + assert "ns/devices/pc/log.json" in github.files diff --git a/crdt-sync/crdt_sync/tests/test_sync_revisions.py b/crdt-sync/crdt_sync/tests/test_sync_revisions.py new file mode 100644 index 0000000..67dde67 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_sync_revisions.py @@ -0,0 +1,434 @@ +"""Tests for the revision tracking that keeps sync inside the free tier. + +Two savings, both measured against the GitHub-backed history this replaces: +88.3% of pushes there were byte-identical no-ops, and every tick +re-downloaded every peer's whole log regardless of whether it had changed. +These tests assert on *request counts*, since that -- not the merge result -- +is what the free-tier headroom depends on. + +Mirrors ``crdt_sync_dart``'s ``test/sync_revisions_test.dart``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from crdt_sync import ( + FileSyncStateStore, + Hlc, + Log, + MemorySyncStateStore, + Record, + SyncState, + default_revs_path, + dump_log, + load_log, + revision_of, + sync_log, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def _log(record_id: str, value: str, node_id: str = "node-a") -> Log: + return { + record_id: Record( + id=record_id, + fields={ + "value": (value, Hlc(wall_time_ms=1000, counter=0, node_id=node_id)) + }, + ) + } + + +# The library's own serialization, so these tests exercise the same encoding +# the apps use rather than a test-only one. +_encode = dump_log +_decode = load_log + + +class FakeRemote: + """An in-memory store that also serves revision maps in one read. + + Stands in for :class:`crdt_sync.FirebaseSyncClient`, counting every call + so tests can assert on traffic rather than only on results. + """ + + def __init__(self, files: dict[str, str] | None = None) -> None: + """Start holding ``files``, with empty call logs.""" + self.files = dict(files or {}) + self.reads: list[str] = [] + self.writes: list[str] = [] + + def list_directory(self, path: str) -> list[str]: + """Return the distinct first segments under ``path``.""" + prefix = f"{path}/" + return sorted( + { + key[len(prefix) :].split("/")[0] + for key in self.files + if key.startswith(prefix) + } + ) + + def get_file_text(self, path: str) -> str | None: + """Return the stored text, recording the read.""" + self.reads.append(path) + return self.files.get(path) + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Store ``text``, recording the write.""" + del message + self.writes.append(path) + self.files[path] = text + + def get_string_map(self, path: str) -> dict[str, str]: + """Return the direct children of ``path`` as a flat map.""" + prefix = f"{path}/" + return { + key[len(prefix) :]: value + for key, value in self.files.items() + if key.startswith(prefix) + } + + def delete_file(self, path: str, *, message: str = "") -> None: + """Remove ``path`` if present.""" + del message + self.files.pop(path, None) + + def can_access_remote(self) -> bool: + """Always reachable.""" + return True + + +class FakeRemoteWithoutBulkRead: + """A store with no bulk-map read, standing in for GitHub. + + Revision tracking must degrade to "fetch everything", not break. + Composes rather than subclasses :class:`FakeRemote`, because a subclass + would inherit ``get_string_map`` -- the very capability this fake exists + to lack. + """ + + def __init__(self, files: dict[str, str] | None = None) -> None: + """Wrap a plain :class:`FakeRemote` holding ``files``.""" + self._inner = FakeRemote(files) + + @property + def files(self) -> dict[str, str]: + """The wrapped store's contents.""" + return self._inner.files + + @property + def reads(self) -> list[str]: + """The wrapped store's read log.""" + return self._inner.reads + + @property + def writes(self) -> list[str]: + """The wrapped store's write log.""" + return self._inner.writes + + def list_directory(self, path: str) -> list[str]: + """Delegate to the wrapped store.""" + return self._inner.list_directory(path) + + def get_file_text(self, path: str) -> str | None: + """Delegate to the wrapped store.""" + return self._inner.get_file_text(path) + + def put_file_text(self, path: str, text: str, *, message: str) -> None: + """Delegate to the wrapped store.""" + self._inner.put_file_text(path, text, message=message) + + def delete_file(self, path: str, *, message: str = "") -> None: + """Delegate to the wrapped store.""" + self._inner.delete_file(path, message=message) + + def can_access_remote(self) -> bool: + """Delegate to the wrapped store.""" + return self._inner.can_access_remote() + + +def _tick( + remote: FakeRemote, + store: object = None, + local: Log | None = None, + device_id: str = "pc", +) -> Log: + return sync_log( + client=remote, + device_id=device_id, + path_prefix="ns/devices", + local_log=local or {}, + encode=_encode, + decode=_decode, + state_store=store, + ) + + +class TestRevisionOf: + def test_is_stable_for_identical_content(self) -> None: + assert revision_of('{"a":1}') == revision_of('{"a":1}') + + def test_differs_for_different_content(self) -> None: + assert revision_of('{"a":1}') != revision_of('{"a":2}') + + +class TestDefaultRevsPath: + def test_is_a_sibling_of_the_device_directory(self) -> None: + assert default_revs_path("diet-guard-sync/devices") == ("diet-guard-sync/revs") + assert default_revs_path("todo-sync/notes") == "todo-sync/revs" + + def test_falls_back_to_a_child_when_there_is_no_parent(self) -> None: + assert default_revs_path("ns") == "ns/revs" + + +class TestSyncState: + def test_round_trips_through_json(self) -> None: + state = SyncState(pushed_rev="abc", peer_revs={"phone": "def"}) + assert SyncState.from_json(state.to_json()) == state + + def test_tolerates_a_missing_peer_map(self) -> None: + assert SyncState.from_json({}).peer_revs == {} + + def test_drops_malformed_peer_entries(self) -> None: + state = SyncState.from_json({"peer_revs": {"phone": 42}}) + assert state.peer_revs == {} + + def test_ignores_a_non_dict_peer_map(self) -> None: + assert SyncState.from_json({"peer_revs": "nonsense"}).peer_revs == {} + + def test_ignores_a_non_string_pushed_rev(self) -> None: + assert SyncState.from_json({"pushed_rev": 42}).pushed_rev is None + + +class TestNoOpPushSuppression: + def test_pushes_and_publishes_a_revision_on_the_first_tick(self) -> None: + remote = FakeRemote() + _tick(remote, MemorySyncStateStore(), _log("a", "1")) + assert remote.writes == ["ns/devices/pc/log.json", "ns/revs/pc"] + + def test_a_second_unchanged_tick_writes_nothing(self) -> None: + remote = FakeRemote() + store = MemorySyncStateStore() + local = _log("a", "1") + _tick(remote, store, local) + remote.writes.clear() + _tick(remote, store, local) + # This is the 88.3% of the old history that was pure waste. + assert not remote.writes + + def test_a_changed_log_pushes_again(self) -> None: + remote = FakeRemote() + store = MemorySyncStateStore() + _tick(remote, store, _log("a", "1")) + remote.writes.clear() + _tick(remote, store, _log("a", "2")) + assert "ns/devices/pc/log.json" in remote.writes + + def test_without_a_state_store_every_tick_pushes(self) -> None: + remote = FakeRemote() + local = _log("a", "1") + _tick(remote, None, local) + _tick(remote, None, local) + assert remote.writes == ["ns/devices/pc/log.json"] * 2 + assert not [w for w in remote.writes if w.startswith("ns/revs")] + + +class TestPeerDownloadSuppression: + def test_downloads_a_peer_whose_revision_it_has_never_seen(self) -> None: + peer = _encode(_log("b", "from-phone", "node-b")) + remote = FakeRemote( + { + "ns/devices/phone/log.json": peer, + "ns/revs/phone": revision_of(peer), + } + ) + merged = _tick(remote, MemorySyncStateStore()) + assert "ns/devices/phone/log.json" in remote.reads + assert "b" in merged + + def test_skips_the_download_when_the_peer_revision_is_unchanged(self) -> None: + peer = _encode(_log("b", "from-phone", "node-b")) + remote = FakeRemote( + { + "ns/devices/phone/log.json": peer, + "ns/revs/phone": revision_of(peer), + } + ) + store = MemorySyncStateStore() + first = _tick(remote, store) + remote.reads.clear() + # The peer's records are already in the local log from tick one, so + # re-downloading is pure waste -- the ~700 MB/month this removes. + _tick(remote, store, first) + assert not remote.reads + + def test_downloads_again_once_the_peer_publishes_a_new_revision(self) -> None: + peer_v1 = _encode(_log("b", "v1", "node-b")) + remote = FakeRemote( + { + "ns/devices/phone/log.json": peer_v1, + "ns/revs/phone": revision_of(peer_v1), + } + ) + store = MemorySyncStateStore() + first = _tick(remote, store) + remote.reads.clear() + + peer_v2 = _encode({**_decode(peer_v1), **_log("c", "v2", "node-b")}) + remote.files["ns/devices/phone/log.json"] = peer_v2 + remote.files["ns/revs/phone"] = revision_of(peer_v2) + + second = _tick(remote, store, first) + assert "ns/devices/phone/log.json" in remote.reads + assert {"b", "c"} <= set(second) + + def test_re_downloads_a_peer_whose_push_was_corrupt(self) -> None: + # A failed decode must not be remembered as seen, or the corruption + # would be permanent. + remote = FakeRemote( + { + "ns/devices/phone/log.json": "not json at all", + "ns/revs/phone": revision_of("not json at all"), + } + ) + store = MemorySyncStateStore() + _tick(remote, store) + remote.reads.clear() + _tick(remote, store) + assert "ns/devices/phone/log.json" in remote.reads + + def test_downloads_when_the_peer_has_published_no_revision(self) -> None: + # A device still running pre-migration code publishes a log but no + # revision; it must not be silently ignored. + peer = _encode(_log("b", "from-phone", "node-b")) + remote = FakeRemote({"ns/devices/phone/log.json": peer}) + assert "b" in _tick(remote, MemorySyncStateStore()) + + def test_skips_a_peer_with_nothing_pushed_yet(self) -> None: + remote = FakeRemote({"ns/devices/phone/other.txt": "x"}) + assert not _tick(remote, MemorySyncStateStore()) + + def test_never_reads_its_own_device_back(self) -> None: + remote = FakeRemote( + { + "ns/devices/pc/log.json": _encode(_log("a", "1")), + "ns/revs/pc": "whatever", + } + ) + _tick(remote, MemorySyncStateStore()) + assert not remote.reads + + +class TestBackendsWithoutBulkMapReads: + def test_still_sync_correctly_just_without_the_saving(self) -> None: + # GitHubSyncClient has no get_string_map, so revision lookup degrades + # to "fetch everything" -- correctness must not depend on it. + peer = _encode(_log("b", "from-phone", "node-b")) + remote = FakeRemoteWithoutBulkRead({"ns/devices/phone/log.json": peer}) + store = MemorySyncStateStore() + first = _tick(remote, store) + assert "b" in first + remote.reads.clear() + _tick(remote, store, first) + assert "ns/devices/phone/log.json" in remote.reads + + +class TestRevisionPublishingOrder: + def test_publishes_the_log_before_its_revision(self) -> None: + # Reversed, a peer would cache "seen rev X" against a log it never + # received, and skip it forever. + remote = FakeRemote() + _tick(remote, MemorySyncStateStore(), _log("a", "1")) + assert remote.writes.index("ns/devices/pc/log.json") < remote.writes.index( + "ns/revs/pc" + ) + + def test_each_device_writes_only_its_own_revision_key(self) -> None: + # Per-device keys rather than one shared map: a whole-map write would + # erase every other device's entry, after which those peers would look + # permanently unchanged and never be fetched again. + remote = FakeRemote({"ns/revs/phone": "peer-rev"}) + _tick(remote, MemorySyncStateStore(), _log("a", "1")) + assert remote.files["ns/revs/phone"] == "peer-rev" + + def test_an_explicit_revs_path_overrides_the_default(self) -> None: + remote = FakeRemote() + sync_log( + client=remote, + device_id="pc", + path_prefix="ns/devices", + local_log=_log("a", "1"), + encode=_encode, + decode=_decode, + state_store=MemorySyncStateStore(), + revs_path="custom/place", + ) + assert "custom/place/pc" in remote.writes + + +class TestTwoDevicesConverge: + def test_each_ends_up_with_the_others_records(self) -> None: + remote = FakeRemote() + pc_store, phone_store = MemorySyncStateStore(), MemorySyncStateStore() + pc = _tick(remote, pc_store, _log("a", "from-pc", "node-pc")) + phone = sync_log( + client=remote, + device_id="phone", + path_prefix="ns/devices", + local_log=_log("b", "from-phone", "node-phone"), + encode=_encode, + decode=_decode, + state_store=phone_store, + ) + assert {"a", "b"} <= set(phone) + pc_after = _tick(remote, pc_store, pc) + assert {"a", "b"} <= set(pc_after) + + +class TestFileSyncStateStore: + def test_round_trips_through_a_file(self, tmp_path: Path) -> None: + store = FileSyncStateStore(tmp_path / "nested" / "state.json") + state = SyncState(pushed_rev="abc", peer_revs={"phone": "def"}) + store.save(state) + assert store.load() == state + + def test_load_returns_default_state_when_absent(self, tmp_path: Path) -> None: + assert FileSyncStateStore(tmp_path / "missing.json").load() == SyncState() + + def test_load_returns_default_state_for_a_truncated_file( + self, tmp_path: Path + ) -> None: + path = tmp_path / "state.json" + path.write_text('{"pushed_rev":', encoding="utf-8") + assert FileSyncStateStore(path).load() == SyncState() + + def test_load_returns_default_state_for_a_non_dict_file( + self, tmp_path: Path + ) -> None: + path = tmp_path / "state.json" + path.write_text("[1, 2, 3]", encoding="utf-8") + assert FileSyncStateStore(path).load() == SyncState() + + def test_survives_across_processes(self, tmp_path: Path) -> None: + # The point of persisting at all: wake_alarm PC is a fresh process + # every minute, so an in-memory store would save nothing. + path = tmp_path / "state.json" + remote = FakeRemote() + local = _log("a", "1") + _tick(remote, FileSyncStateStore(path), local) + remote.writes.clear() + _tick(remote, FileSyncStateStore(path), local) + assert not remote.writes + + +@pytest.mark.parametrize("device_id", ["pc", "phone"]) +def test_a_device_publishes_its_revision_under_its_own_id(device_id: str) -> None: + remote = FakeRemote() + _tick(remote, MemorySyncStateStore(), _log("a", "1"), device_id=device_id) + assert f"ns/revs/{device_id}" in remote.writes diff --git a/crdt-sync/pyproject.toml b/crdt-sync/pyproject.toml index 6d26293..1ae9fc5 100644 --- a/crdt-sync/pyproject.toml +++ b/crdt-sync/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "crdt-sync" -version = "0.3.0" +version = "0.4.0" description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps" requires-python = ">=3.10" dependencies = ["requests"] diff --git a/crdt_sync_dart/lib/crdt_sync.dart b/crdt_sync_dart/lib/crdt_sync.dart index 6a934fb..7b0b49f 100644 --- a/crdt_sync_dart/lib/crdt_sync.dart +++ b/crdt_sync_dart/lib/crdt_sync.dart @@ -1,9 +1,13 @@ -/// Shared CRDT merge scheme + GitHub-Contents-API sync transport. +/// Shared CRDT merge scheme + pluggable remote sync transports. library; +export 'src/firebase_auth_rest.dart'; +export 'src/firebase_client.dart'; export 'src/github_client.dart'; export 'src/hlc.dart'; export 'src/log.dart'; +export 'src/mirror_store.dart'; export 'src/record.dart'; +export 'src/remote_store.dart'; export 'src/store.dart'; export 'src/sync.dart'; diff --git a/crdt_sync_dart/lib/src/firebase_auth_rest.dart b/crdt_sync_dart/lib/src/firebase_auth_rest.dart new file mode 100644 index 0000000..c35ad96 --- /dev/null +++ b/crdt_sync_dart/lib/src/firebase_auth_rest.dart @@ -0,0 +1,250 @@ +/// Firebase Authentication over its REST API, with no FlutterFire dependency. +/// +/// The official `firebase_auth` plugin ships iOS/Android/Web/macOS only -- +/// there is no Linux desktop support -- but these apps run on Linux desktop +/// *and* Android, and one (`wake_alarm`) runs headless under systemd. The +/// REST endpoints are plain HTTPS and behave identically everywhere, so this +/// is the only auth path that covers every target. +library; + +import 'dart:convert'; + +import 'package:http/http.dart' as http; + +import 'remote_store.dart'; + +const _signInBase = 'https://identitytoolkit.googleapis.com/v1'; +const _refreshBase = 'https://securetoken.googleapis.com/v1'; + +/// Refresh this long before the token actually expires. +/// +/// An ID token lives ~1 h. Refreshing early means a tick that starts just +/// under the wire still finishes with a valid token, rather than 401ing +/// halfway through a multi-file push. +const _refreshSkew = Duration(minutes: 5); + +/// Raised for an authentication failure the caller must not silently ignore. +/// +/// A [RemoteSyncError] so callers that only care about "sync is broken" can +/// catch one type, while a settings screen can single this out to say +/// "your password is wrong" rather than "the network is down". +class FirebaseAuthError extends RemoteSyncError { + FirebaseAuthError(super.message); +} + +/// One device's Firebase session: a short-lived ID token plus the long-lived +/// refresh token that mints the next one. +class FirebaseCredentials { + FirebaseCredentials({ + required this.idToken, + required this.refreshToken, + required this.expiresAt, + }); + + factory FirebaseCredentials.fromJson(Map json) => + FirebaseCredentials( + idToken: json['id_token'] as String, + refreshToken: json['refresh_token'] as String, + expiresAt: DateTime.parse(json['expires_at'] as String).toUtc(), + ); + + /// The bearer credential for Realtime Database requests. Expires quickly. + final String idToken; + + /// The long-lived credential. **This is the secret worth protecting** -- + /// store it in `flutter_secure_storage` on Android or a `0600` file on + /// Linux, never in plain `SharedPreferences`. + final String refreshToken; + + final DateTime expiresAt; + + /// Whether [idToken] is expired, or close enough that a tick starting now + /// might outlive it. + bool isExpiredAt(DateTime now) => !now.add(_refreshSkew).isBefore(expiresAt); + + Map toJson() => { + 'id_token': idToken, + 'refresh_token': refreshToken, + 'expires_at': expiresAt.toUtc().toIso8601String(), + }; +} + +/// Where a device keeps its [FirebaseCredentials] between runs. +/// +/// Abstract because the right answer differs per platform, and one of the +/// callers (`wake_alarm` PC) is a **fresh process every minute** -- without +/// persistence it would re-authenticate 1440 times a day. +abstract interface class FirebaseCredentialStore { + Future load(); + Future save(FirebaseCredentials credentials); + Future clear(); +} + +/// An in-memory [FirebaseCredentialStore], for tests and one-shot scripts. +class InMemoryCredentialStore implements FirebaseCredentialStore { + InMemoryCredentialStore([this._credentials]); + + FirebaseCredentials? _credentials; + + @override + Future load() async => _credentials; + + @override + Future save(FirebaseCredentials credentials) async => + _credentials = credentials; + + @override + Future clear() async => _credentials = null; +} + +/// Signs in and keeps a valid ID token available, refreshing as needed. +/// +/// [apiKey] is the project's public Web API key -- not a secret, and safe to +/// ship inside an APK. The actual credential is the refresh token held by +/// [store]. A service-account key is deliberately **not** used: it would +/// bypass the security rules entirely and is trivially extractable from an +/// installed app. +class FirebaseTokenProvider { + FirebaseTokenProvider({ + required this.apiKey, + required this.store, + http.Client? httpClient, + DateTime Function()? clock, + }) : _http = httpClient ?? http.Client(), + _clock = clock ?? DateTime.now; + + final String apiKey; + + /// Where the refresh token lives between runs. Public so a settings screen + /// can swap in the platform-appropriate backing store. + final FirebaseCredentialStore store; + final http.Client _http; + final DateTime Function() _clock; + + FirebaseCredentials? _cached; + + /// Exchanges an email/password for a session and persists it. + /// + /// Called once per device from a settings screen or a setup command; every + /// later run reuses the stored refresh token. + Future signIn({ + required String email, + required String password, + }) async { + final body = await _post( + Uri.parse('$_signInBase/accounts:signInWithPassword?key=$apiKey'), + {'email': email, 'password': password, 'returnSecureToken': true}, + 'sign in', + ); + await _adopt( + idToken: body['idToken'] as String, + refreshToken: body['refreshToken'] as String, + expiresInSeconds: body['expiresIn'] as String, + ); + } + + /// Returns a currently-valid ID token, refreshing or failing loudly. + /// + /// Throws [FirebaseAuthError] when no session is stored or the refresh + /// token has been revoked -- never returns a stale token and never + /// silently no-ops, because a sync that quietly stops syncing is the + /// failure mode this whole design is trying to avoid. + Future idToken() async { + final credentials = _cached ??= await store.load(); + if (credentials == null) { + throw FirebaseAuthError( + 'not signed in: no stored refresh token for this device', + ); + } + if (!credentials.isExpiredAt(_clock())) return credentials.idToken; + return _refresh(credentials.refreshToken); + } + + /// Whether this device has a stored session at all. + /// + /// Distinguishes "sync is not configured" (fine -- `screen-locker` treats a + /// missing credential as a normal state) from "sync is configured and + /// broken" (an error). + Future hasSession() async => + (_cached ??= await store.load()) != null; + + /// Forgets the stored session, so the next [idToken] fails loudly. + Future signOut() async { + _cached = null; + await store.clear(); + } + + Future _refresh(String refreshToken) async { + final body = await _post( + Uri.parse('$_refreshBase/token?key=$apiKey'), + {'grant_type': 'refresh_token', 'refresh_token': refreshToken}, + 'refresh the session', + ); + return _adopt( + idToken: body['id_token'] as String, + // A refresh may hand back a rotated refresh token; keep the new one. + refreshToken: body['refresh_token'] as String, + expiresInSeconds: body['expires_in'] as String, + ); + } + + Future _adopt({ + required String idToken, + required String refreshToken, + required String expiresInSeconds, + }) async { + final credentials = FirebaseCredentials( + idToken: idToken, + refreshToken: refreshToken, + expiresAt: _clock().toUtc().add( + Duration(seconds: int.parse(expiresInSeconds)), + ), + ); + _cached = credentials; + await store.save(credentials); + return idToken; + } + + Future> _post( + Uri uri, + Map payload, + String what, + ) async { + late final http.Response res; + try { + res = await _http.post( + uri, + headers: {'Content-Type': 'application/json'}, + body: jsonEncode(payload), + ); + } on http.ClientException catch (exc) { + throw FirebaseAuthError('network error trying to $what: $exc'); + } + if (res.statusCode < 200 || res.statusCode >= 300) { + throw FirebaseAuthError( + 'failed to $what: HTTP ${res.statusCode} ${_reason(res.body)}', + ); + } + return jsonDecode(res.body) as Map; + } + + /// Pulls Google's machine-readable reason (`INVALID_PASSWORD`, + /// `TOKEN_EXPIRED`, `USER_DISABLED`, ...) out of an error body, so the + /// thrown message says what actually went wrong instead of just "400". + static String _reason(String body) { + try { + final decoded = jsonDecode(body); + if (decoded is Map) { + final error = decoded['error']; + if (error is Map) return '(${error['message']})'; + if (error is String) return '($error)'; + } + } on FormatException { + // Non-JSON body (a proxy error page, say): the status code is all the + // detail there is. + } + return ''; + } + + void close() => _http.close(); +} diff --git a/crdt_sync_dart/lib/src/firebase_client.dart b/crdt_sync_dart/lib/src/firebase_client.dart new file mode 100644 index 0000000..4660838 --- /dev/null +++ b/crdt_sync_dart/lib/src/firebase_client.dart @@ -0,0 +1,283 @@ +/// Firebase Realtime Database as dumb keyed storage, over its REST API. +/// +/// The RTDB REST endpoints are plain HTTPS, so this works identically on +/// Linux desktop, Android and headless systemd -- unlike `firebase_database`, +/// which has no Linux desktop support at all. +/// +/// Why RTDB rather than Firestore: on the Spark (free) plan RTDB bills only +/// storage and bandwidth, with **no per-operation quota**, so a misbehaving +/// sync loop can never exhaust a daily budget and silently stop working +/// mid-day. Its path model is also a direct match for the existing +/// `//` layout. +library; + +import 'dart:convert'; + +import 'package:http/http.dart' as http; + +import 'firebase_auth_rest.dart'; +import 'remote_store.dart'; + +const _httpUnauthorized = 401; +const _httpForbidden = 403; + +/// Characters Realtime Database forbids in a key, plus `~` itself because it +/// is this module's escape character. +/// +/// `/` is absent deliberately: it is the path separator, so it is handled by +/// splitting into segments before any segment is escaped. +const _escapedChars = { + '~': '~7E', + '.': '~2E', + r'$': '~24', + '#': '~23', + '[': '~5B', + ']': '~5D', +}; + +/// Raised for a Realtime Database failure the caller must not silently +/// ignore. +class FirebaseSyncError extends RemoteSyncError { + FirebaseSyncError(super.message); +} + +/// Raised when the database itself is unreachable or the credential is not +/// authorized for it. +/// +/// The Firebase counterpart of `RepoNotFoundError`: it means "the database +/// URL is wrong, or the security rules reject this uid", as opposed to +/// "nothing has been pushed to that path yet", which is benign and surfaces +/// as a null/empty result. +class DatabaseNotFoundError extends FirebaseSyncError + implements RemoteNotFoundError { + DatabaseNotFoundError(super.message); +} + +/// Escapes one path segment into a legal Realtime Database key. +/// +/// RTDB rejects `. $ # [ ] /` in keys, and the REST API's trailing `.json` +/// is a *format suffix* rather than part of the path -- so a filename like +/// `log.json` cannot be stored verbatim. The mapping is a reversible `~XX` +/// hex escape (`log.json` -> `log~2Ejson`) rather than a lossy "strip the +/// extension", because callers see these names again: todo-app's +/// `listDirectory('todo-sync/notes')` returns *filenames*, not device +/// directories, and must keep getting `.json` back. +/// +/// `~` is used as the escape character because it is legal in RTDB keys and +/// unreserved in URLs, so the escaped form needs no percent-encoding -- which +/// would otherwise be decoded back into the illegal character by the server. +String encodeKey(String segment) { + final buffer = StringBuffer(); + for (final rune in segment.split('')) { + buffer.write(_escapedChars[rune] ?? rune); + } + return buffer.toString(); +} + +/// Reverses [encodeKey]. +String decodeKey(String key) { + final buffer = StringBuffer(); + var index = 0; + while (index < key.length) { + final escape = key.length - index >= 3 ? key.substring(index, index + 3) : ''; + final decoded = _escapeToChar[escape]; + if (decoded == null) { + buffer.write(key[index]); + index += 1; + } else { + buffer.write(decoded); + index += 3; + } + } + return buffer.toString(); +} + +final Map _escapeToChar = { + for (final entry in _escapedChars.entries) entry.value: entry.key, +}; + +/// Escapes every segment of a `/`-separated logical path. +String encodePath(String path) => + path.split('/').where((s) => s.isNotEmpty).map(encodeKey).join('/'); + +/// Realtime Database seen through the same three-method contract as GitHub. +/// +/// Reads and writes UTF-8 text blobs stored as JSON string leaves, so the +/// on-the-wire payload is byte-identical to what the GitHub backend stored +/// and the two can be mirrored against each other during migration. +class FirebaseRestClient implements RemoteStore, BulkMapReader { + FirebaseRestClient({ + required String databaseUrl, + required this.auth, + http.Client? httpClient, + }) : _databaseUrl = databaseUrl.endsWith('/') + ? databaseUrl.substring(0, databaseUrl.length - 1) + : databaseUrl, + _http = httpClient ?? http.Client(); + + final String _databaseUrl; + + /// Mints the ID token every request carries. Public so a settings screen + /// can drive `signIn` / `signOut` through the same instance the client uses. + final FirebaseTokenProvider auth; + final http.Client _http; + + Future _uri(String path, {Map query = const {}}) async { + final encoded = encodePath(path); + return Uri.parse('$_databaseUrl/$encoded.json').replace( + queryParameters: {'auth': await auth.idToken(), ...query}, + ); + } + + /// Turns a non-2xx response into the right error type. + /// + /// 401/403 mean the rules rejected this uid or the token is bad -- the + /// database-level failure a caller may want to single out. Everything else + /// (including a Spark quota exhaustion, which answers with an error rather + /// than billing you) is a plain [FirebaseSyncError]. Nothing here is ever + /// swallowed: a quota-exhausted database that silently returned "no data" + /// would look exactly like "nothing to sync". + Never _raise(String what, http.Response res) { + final detail = res.body.isEmpty ? '' : ' ${res.body.trim()}'; + if (res.statusCode == _httpUnauthorized || + res.statusCode == _httpForbidden) { + throw DatabaseNotFoundError( + '$what rejected: HTTP ${res.statusCode}$detail -- the database URL ' + 'or the security rules do not allow this account', + ); + } + throw FirebaseSyncError('$what failed: HTTP ${res.statusCode}$detail'); + } + + Future _get( + String path, { + Map query = const {}, + }) async { + try { + return await _http.get(await _uri(path, query: query)); + } on http.ClientException catch (exc) { + throw FirebaseSyncError('network error reading $path: $exc'); + } + } + + @override + Future> listDirectory(String path) async { + // shallow=true returns `{key: true}` without any values, so listing a + // directory costs bytes rather than the whole subtree. This is the + // difference between a sync tick costing hundreds of bytes and costing + // hundreds of kilobytes. + final res = await _get(path, query: {'shallow': 'true'}); + if (res.statusCode < 200 || res.statusCode >= 300) { + _raise('listing $path', res); + } + final decoded = jsonDecode(res.body); + if (decoded is! Map) return []; + return decoded.keys.map(decodeKey).toList(); + } + + @override + Future getFileText(String path) async { + final res = await _get(path); + if (res.statusCode < 200 || res.statusCode >= 300) { + _raise('reading $path', res); + } + final decoded = jsonDecode(res.body); + if (decoded == null) return null; + if (decoded is! String) { + throw FirebaseSyncError( + 'value at $path is ${decoded.runtimeType}, not the expected text blob', + ); + } + return decoded; + } + + /// Writes [text] at [path]. [message] is ignored -- Realtime Database has + /// no commit log to attach a reason to. + @override + Future putFileText( + String path, + String text, { + required String message, + }) async { + late final http.Response res; + try { + res = await _http.put(await _uri(path), body: jsonEncode(text)); + } on http.ClientException catch (exc) { + throw FirebaseSyncError('network error writing $path: $exc'); + } + if (res.statusCode < 200 || res.statusCode >= 300) { + _raise('writing $path', res); + } + } + + /// Merges [values] into the map at [path], leaving sibling keys alone. + /// + /// Used for the shared `revs` map, where a plain PUT would replace the + /// whole map and wipe every *other* device's entry -- after which those + /// devices would look permanently unchanged and never be fetched again. + Future patchValues(String path, Map values) async { + late final http.Response res; + try { + res = await _http.patch(await _uri(path), body: jsonEncode(values)); + } on http.ClientException catch (exc) { + throw FirebaseSyncError('network error patching $path: $exc'); + } + if (res.statusCode < 200 || res.statusCode >= 300) { + _raise('patching $path', res); + } + } + + /// Reads the map at [path] as `key -> string`, or an empty map if absent. + /// + /// Tolerates a non-map or malformed value rather than throwing: this backs + /// the revision cache, which is an *optimisation*. A corrupt revs node must + /// degrade into "fetch everything", never into a failed sync. + @override + Future> getStringMap(String path) async { + final res = await _get(path); + if (res.statusCode < 200 || res.statusCode >= 300) { + _raise('reading $path', res); + } + final decoded = jsonDecode(res.body); + if (decoded is! Map) return {}; + return { + for (final entry in decoded.entries) + if (entry.value is String) decodeKey(entry.key): entry.value as String, + }; + } + + @override + Future deleteFile( + String path, { + String message = 'crdt_sync: delete', + }) async { + late final http.Response res; + try { + res = await _http.delete(await _uri(path)); + } on http.ClientException catch (exc) { + throw FirebaseSyncError('network error deleting $path: $exc'); + } + if (res.statusCode < 200 || res.statusCode >= 300) { + _raise('deleting $path', res); + } + } + + /// Probes the database root. Never throws -- a bad URL, a rejected token or + /// a network failure all report false. + @override + Future canAccessRemote() async { + try { + final res = await _http.get(await _uri('', query: {'shallow': 'true'})); + return res.statusCode >= 200 && res.statusCode < 300; + } on http.ClientException { + return false; + } on RemoteSyncError { + // Includes FirebaseAuthError from the token refresh: "cannot get a + // token" is exactly "cannot access the remote". + return false; + } + } + + @override + void close() => _http.close(); +} diff --git a/crdt_sync_dart/lib/src/github_client.dart b/crdt_sync_dart/lib/src/github_client.dart index 70b491e..c26fe7b 100644 --- a/crdt_sync_dart/lib/src/github_client.dart +++ b/crdt_sync_dart/lib/src/github_client.dart @@ -2,17 +2,14 @@ import 'dart:convert'; import 'package:http/http.dart' as http; +import 'remote_store.dart'; + const _apiBase = 'https://api.github.com'; const _httpNotFound = 404; /// Raised for a GitHub API failure the caller must not silently ignore. -class GitHubSyncError implements Exception { - GitHubSyncError(this.message); - - final String message; - - @override - String toString() => 'GitHubSyncError: $message'; +class GitHubSyncError extends RemoteSyncError { + GitHubSyncError(super.message); } /// Raised when the configured repo itself is unreachable. @@ -21,7 +18,12 @@ class GitHubSyncError implements Exception { /// benign -- it just means no other device has synced before) so the caller /// can tell "the repo name is wrong or the token isn't scoped to it" apart /// from "no other device has synced yet". -class RepoNotFoundError extends GitHubSyncError { +/// +/// Extends [GitHubSyncError] so existing `on GitHubSyncError` catches still +/// cover it, and implements [RemoteNotFoundError] so backend-neutral callers +/// can catch "the remote itself is missing" without naming GitHub. Dart's +/// single inheritance is why one is `extends` and the other `implements`. +class RepoNotFoundError extends GitHubSyncError implements RemoteNotFoundError { RepoNotFoundError(super.message); } @@ -36,7 +38,7 @@ class RepoNotFoundError extends GitHubSyncError { /// `//` layout needs to discover device /// *subdirectories*; and [putFileText] resolves its own current SHA rather /// than requiring the caller to track it. -class GitHubClient { +class GitHubClient implements RemoteStore { GitHubClient({ required this.owner, required this.repo, @@ -94,6 +96,7 @@ class GitHubClient { /// /// Includes both files and subdirectories -- see the class doc for why /// this differs from a plain "list files" helper. + @override Future> listDirectory(String path) async { final res = await _get(path); if (res.statusCode == _httpNotFound) { @@ -113,6 +116,7 @@ class GitHubClient { } /// Returns the decoded text content at [path], or null if unused. + @override Future getFileText(String path) async { final res = await _get(path); if (res.statusCode == _httpNotFound) { @@ -141,6 +145,7 @@ class GitHubClient { } /// Creates or updates the file at [path] with [text]. + @override Future putFileText( String path, String text, { @@ -172,10 +177,17 @@ class GitHubClient { /// A lightweight connection test for a settings "Test connection" button: it /// hits the bare repo endpoint, so it succeeds even before any file has been /// pushed. Never throws -- a network failure or missing repo returns false. + /// + /// The GitHub-era name for [canAccessRemote]; kept because app settings + /// screens call it. Both do the same probe. Future canAccessRepo() => _repoExists(); + @override + Future canAccessRemote() => _repoExists(); + /// Deletes the file at [path], resolving its current sha itself so callers /// don't have to track it. A no-op if [path] does not exist. + @override Future deleteFile( String path, { String message = 'crdt_sync: delete', @@ -197,5 +209,6 @@ class GitHubClient { } } + @override void close() => _http.close(); } diff --git a/crdt_sync_dart/lib/src/mirror_store.dart b/crdt_sync_dart/lib/src/mirror_store.dart new file mode 100644 index 0000000..2261d36 --- /dev/null +++ b/crdt_sync_dart/lib/src/mirror_store.dart @@ -0,0 +1,133 @@ +/// A [RemoteStore] that writes to two backends and reads from both. +/// +/// Exists for the GitHub -> Firebase cutover: each app can move to Firebase +/// while still mirroring to the old repo, so rolling back is a constructor +/// change rather than a data recovery. +library; + +import 'remote_store.dart'; + +/// Called when the mirror backend fails, so the failure is loud but survivable. +typedef MirrorFailureHandler = + void Function(String operation, Object error); + +/// Writes to both backends; reads from both and prefers the primary. +/// +/// **Reads consult both on purpose.** Each app spans two devices, and they +/// cut over one at a time. If reads were primary-only, a migrated desktop +/// would never see an un-migrated phone's writes while still mirroring its +/// own back -- silent one-directional convergence, with no error raised. +/// Because `mergeLogs` is commutative and idempotent, a union read costs +/// nothing semantically and makes the cutover order irrelevant. +/// +/// Write asymmetry is deliberate too: the primary is authoritative, so a +/// primary failure fails the tick (fail-closed), while a mirror failure is +/// reported through [onMirrorFailure] and otherwise tolerated. Once the +/// mirror is retired, the old backend going away must not break sync. +class MirrorStore implements RemoteStore, BulkMapReader { + MirrorStore({ + required this.primary, + required this.mirror, + this.onMirrorFailure, + }); + + /// The authoritative backend. Its failures fail the tick. + final RemoteStore primary; + + /// The backup backend. Kept in step, but never allowed to fail a tick. + final RemoteStore mirror; + + /// Notified when the mirror misbehaves. A silent mirror failure would let + /// the fallback rot unnoticed until the day it was needed. + final MirrorFailureHandler? onMirrorFailure; + + @override + Future> listDirectory(String path) async { + final names = {...await primary.listDirectory(path)}; + await _tryMirror('listDirectory $path', () async { + names.addAll(await mirror.listDirectory(path)); + }); + return names.toList(); + } + + @override + Future getFileText(String path) async { + final text = await primary.getFileText(path); + if (text != null) return text; + // Absent from the primary means "this device has not migrated yet", not + // "no data" -- so fall through rather than reporting nothing. + String? fallback; + await _tryMirror('getFileText $path', () async { + fallback = await mirror.getFileText(path); + }); + return fallback; + } + + @override + Future putFileText( + String path, + String text, { + required String message, + }) async { + await primary.putFileText(path, text, message: message); + await _tryMirror('putFileText $path', () async { + await mirror.putFileText(path, text, message: message); + }); + } + + @override + Future deleteFile( + String path, { + String message = 'crdt_sync: delete', + }) async { + await primary.deleteFile(path, message: message); + await _tryMirror('deleteFile $path', () async { + await mirror.deleteFile(path, message: message); + }); + } + + /// Whether the **primary** is reachable. + /// + /// Only the primary: a settings "Test connection" button must not report + /// success because the backend being retired happens to answer. + @override + Future canAccessRemote() => primary.canAccessRemote(); + + /// Merges both backends' revision maps, preferring the primary's entry. + /// + /// A device that has not migrated publishes revisions only to the mirror; + /// without this it would look revision-less and be re-downloaded every + /// tick for the whole trial period. + @override + Future> getStringMap(String path) async { + final merged = {}; + // Pattern binds rather than `is` promotion: BulkMapReader is not a + // subtype of RemoteStore, and Dart only promotes to subtypes of the + // declared type, so the intersection has to be named explicitly. + await _tryMirror('getStringMap $path', () async { + if (mirror case final BulkMapReader reader) { + merged.addAll(await reader.getStringMap(path)); + } + }); + // Primary applied last so its entries win on conflict. + if (primary case final BulkMapReader reader) { + merged.addAll(await reader.getStringMap(path)); + } + return merged; + } + + @override + void close() { + primary.close(); + mirror.close(); + } + + /// Runs a mirror operation, reporting rather than propagating its failure. + Future _tryMirror(String operation, Future Function() run) async { + try { + await run(); + } on RemoteSyncError catch (error) { + onMirrorFailure?.call(operation, error); + } + } +} diff --git a/crdt_sync_dart/lib/src/remote_store.dart b/crdt_sync_dart/lib/src/remote_store.dart new file mode 100644 index 0000000..d7acbd2 --- /dev/null +++ b/crdt_sync_dart/lib/src/remote_store.dart @@ -0,0 +1,88 @@ +/// Raised for a remote-storage failure the caller must not silently ignore. +/// +/// The backend-neutral base of the error hierarchy: catch this to handle +/// "the sync transport failed" regardless of which backend is configured. +/// Backends narrow it -- `GitHubSyncError` and `FirebaseSyncError` both +/// extend it -- so existing `on GitHubSyncError` catches keep working while a +/// caller that has been migrated can catch the base type instead. +class RemoteSyncError implements Exception { + RemoteSyncError(this.message); + + final String message; + + // Uses runtimeType so subclasses read as themselves without each having to + // override toString. + @override + String toString() => '$runtimeType: $message'; +} + +/// Raised when the configured remote itself is unreachable. +/// +/// Distinguished from a missing *path* (nothing pushed there yet, which is +/// benign -- it just means no other device has synced before) so the caller +/// can tell "the repo/database is wrong or the credential isn't scoped to it" +/// apart from "no other device has synced yet". +class RemoteNotFoundError extends RemoteSyncError { + RemoteNotFoundError(super.message); +} + +/// The whole remote surface `crdt_sync` syncs through: dumb keyed storage of +/// UTF-8 text blobs, with directory listing. +/// +/// Deliberately tiny. `syncLog` and every app-level sync service talk only to +/// this, so swapping the storage backend (GitHub Contents API, Firebase +/// Realtime Database, a dual-writing mirror of both) is a constructor change +/// at the call site and nothing more. +/// +/// Implementations are expected to throw [RemoteSyncError] (or a subclass) for +/// any failure the caller must not silently ignore, and to treat a missing +/// path as a benign `null` / empty rather than an error. +abstract interface class RemoteStore { + /// Returns the entry names directly under [path], or an empty list when + /// nothing has been written there yet. + /// + /// Includes both files and subdirectories: the + /// `//` layout needs to discover device + /// *directories*, not just files. + Future> listDirectory(String path); + + /// Returns the text stored at [path], or null when nothing is there yet. + Future getFileText(String path); + + /// Creates or replaces the text at [path]. + /// + /// [message] is a human-readable reason for the write. Backends that record + /// one (GitHub commits) use it; backends that don't (Firebase) ignore it. + Future putFileText(String path, String text, {required String message}); + + /// Deletes [path]. A no-op when [path] does not exist. + Future deleteFile(String path, {String message}); + + /// Returns whether the configured credential can reach the remote. + /// + /// A lightweight connection test for a settings "Test connection" button: it + /// probes the remote root, so it succeeds even before any file has been + /// pushed. Never throws -- a network failure or missing remote returns + /// false. + Future canAccessRemote(); + + /// Releases the underlying HTTP client. + void close(); +} + +/// Optional capability: read a whole small map of text values in one request. +/// +/// Implemented by backends where that is a single cheap call +/// (`FirebaseRestClient`), and absent where it would be one request per key +/// (`GitHubClient`). [syncLog] uses it to fetch every peer's revision in one +/// go and skip downloading logs that have not changed -- the difference +/// between a sync tick costing a few hundred bytes and a few hundred +/// kilobytes. A backend that does not implement it simply syncs the old way. +abstract interface class BulkMapReader { + /// Returns the map at [path] as `key -> text`, empty when absent. + /// + /// Never throws for a malformed or missing value: this backs an + /// *optimisation*, and a corrupt map must degrade into "fetch everything", + /// never into a failed sync. + Future> getStringMap(String path); +} diff --git a/crdt_sync_dart/lib/src/sync.dart b/crdt_sync_dart/lib/src/sync.dart index ab4cb5c..b502f98 100644 --- a/crdt_sync_dart/lib/src/sync.dart +++ b/crdt_sync_dart/lib/src/sync.dart @@ -1,13 +1,86 @@ -import 'github_client.dart'; +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; + import 'log.dart'; import 'record.dart'; +import 'remote_store.dart'; const _defaultFilename = 'log.json'; +/// What one device remembers between ticks so it can skip needless traffic. +/// +/// Two facts, both cheap: +/// * [pushedRev] -- the hash of what this device last pushed, so an unchanged +/// log is not re-uploaded. 88% of the pushes in the GitHub-backed history +/// were byte-identical no-ops. +/// * [peerRevs] -- the hash each peer had last time we downloaded it, so an +/// unchanged peer is not re-downloaded. +/// +/// **Must be stored next to the log itself and cleared with it.** Skipping an +/// unchanged peer is only sound because that peer's records are already +/// merged into the local log; a cache that outlived its log would skip peers +/// whose data had been lost. +class SyncState { + const SyncState({this.pushedRev, this.peerRevs = const {}}); + + factory SyncState.fromJson(Map json) => SyncState( + pushedRev: json['pushed_rev'] as String?, + peerRevs: { + for (final entry + in (json['peer_revs'] as Map? ?? {}).entries) + if (entry.value is String) entry.key: entry.value as String, + }, + ); + + final String? pushedRev; + final Map peerRevs; + + Map toJson() => { + 'pushed_rev': pushedRev, + 'peer_revs': peerRevs, + }; +} + +/// Where [syncLog] persists its [SyncState] between runs. +/// +/// Persistence is what makes the saving real for the short-lived callers: +/// `wake_alarm` PC is a fresh process every minute and `diet_guard` PC a +/// fresh process every 15 minutes, so an in-memory cache would save nothing. +abstract interface class SyncStateStore { + Future load(); + Future save(SyncState state); +} + +/// A [SyncStateStore] that forgets everything, for tests and one-shot runs. +/// +/// Correct but pessimistic: every tick re-downloads every peer and re-pushes +/// the local log, which is exactly the old behaviour. +class InMemorySyncStateStore implements SyncStateStore { + SyncState _state = const SyncState(); + + @override + Future load() async => _state; + + @override + Future save(SyncState state) async => _state = state; +} + +/// The revision of a serialized log: a content hash, not a clock reading. +/// +/// A hash rather than the log's maximum HLC because merging a peer's *older* +/// record changes the content without raising the maximum -- with three +/// devices in one namespace (diet-guard has `pc`, `phone` and `desktop`) a +/// clock-based revision can miss a peer's merged state. It is also the same +/// value used to suppress no-op pushes, so the two optimisations share one +/// mechanism. +String revisionOf(String encodedLog) => + sha256.convert(utf8.encode(encodedLog)).toString(); + /// Runs one full sync tick: pull every other device's log, merge, push. /// /// Pulls from `//` for every device -/// directory GitHub reports under [pathPrefix], merges each into +/// directory the remote reports under [pathPrefix], merges each into /// [localLog] with [mergeLogs], then pushes this device's own merged result /// to `//`. /// @@ -18,8 +91,20 @@ const _defaultFilename = 'log.json'; /// corrupt/unparsable push, and that device is skipped for this tick /// rather than aborting the whole sync. Mirrors the Python `sync_log`'s /// `(ValueError, KeyError, TypeError)` catch for the same reason. +/// +/// Pass [stateStore] to enable revision tracking, which skips downloading +/// peers that have not changed and skips pushing a log that has not changed. +/// Without it the tick behaves exactly as it always did: fetch everything, +/// push unconditionally. +/// +/// Revisions live at `/` -- one small text node per +/// device, each written only by its owner. Deliberately *not* one shared map +/// per namespace: a whole-map write would erase every other device's entry, +/// after which those peers would look permanently unchanged and never be +/// fetched again. Per-device keys make that failure impossible rather than +/// merely avoided. Future syncLog({ - required GitHubClient client, + required RemoteStore client, required String deviceId, required String pathPrefix, required Log localLog, @@ -27,10 +112,27 @@ Future syncLog({ required Log Function(String text) decode, String filename = _defaultFilename, String commitMessage = 'crdt_sync: update log', + SyncStateStore? stateStore, + String? revsPath, }) async { + final revs = revsPath ?? defaultRevsPath(pathPrefix); + final state = await stateStore?.load() ?? const SyncState(); + final remoteRevs = stateStore == null + ? const {} + : await _remoteRevs(client, revs); + var mergedLog = Map.from(localLog); + final seenRevs = {}; for (final otherDeviceId in await client.listDirectory(pathPrefix)) { if (otherDeviceId == deviceId) continue; + final remoteRev = remoteRevs[otherDeviceId]; + if (remoteRev != null && remoteRev == state.peerRevs[otherDeviceId]) { + // Unchanged since we last merged it, and that merge is already part of + // localLog -- so the (potentially hundreds of KB) download is pure + // waste. Carry the revision forward so it stays skipped next tick. + seenRevs[otherDeviceId] = remoteRev; + continue; + } final text = await client.getFileText( '$pathPrefix/$otherDeviceId/$filename', ); @@ -42,12 +144,56 @@ Future syncLog({ } on TypeError { continue; } + // Only remembered once the merge succeeded: a corrupt push must be + // retried next tick, not marked as seen. + seenRevs[otherDeviceId] = remoteRev ?? revisionOf(text); } - await client.putFileText( - '$pathPrefix/$deviceId/$filename', - encode(mergedLog), - message: commitMessage, - ); + final encoded = encode(mergedLog); + final rev = revisionOf(encoded); + final unchanged = stateStore != null && rev == state.pushedRev; + if (!unchanged) { + await client.putFileText( + '$pathPrefix/$deviceId/$filename', + encoded, + message: commitMessage, + ); + if (stateStore != null) { + // Published after the log, never before: a peer that cached "seen rev + // X" against a log it never received would skip it forever. + await client.putFileText( + '$revs/$deviceId', + rev, + message: commitMessage, + ); + } + } + await stateStore?.save(SyncState(pushedRev: rev, peerRevs: seenRevs)); return mergedLog; } + +/// Where revisions live for a given [pathPrefix]: a `revs` sibling of the +/// device directory, e.g. `diet-guard-sync/devices` -> `diet-guard-sync/revs` +/// and `todo-sync/notes` -> `todo-sync/revs`. +String defaultRevsPath(String pathPrefix) { + final cut = pathPrefix.lastIndexOf('/'); + return cut < 0 ? '$pathPrefix/revs' : '${pathPrefix.substring(0, cut)}/revs'; +} + +/// Reads every peer's published revision, cheaply where the backend allows. +/// +/// Degrades to an empty map -- meaning "fetch everything", the old behaviour +/// -- on any backend without [BulkMapReader], so correctness never depends on +/// the optimisation being available. +Future> _remoteRevs( + RemoteStore client, + String revsPath, +) async { + // A pattern bind rather than `is!` + promotion: BulkMapReader is not a + // subtype of RemoteStore, and Dart only promotes to subtypes of the + // declared type -- so the intersection has to be named explicitly. + if (client case final BulkMapReader reader) { + return reader.getStringMap(revsPath); + } + return const {}; +} diff --git a/crdt_sync_dart/pubspec.yaml b/crdt_sync_dart/pubspec.yaml index 05ba1b0..d2fcdb4 100644 --- a/crdt_sync_dart/pubspec.yaml +++ b/crdt_sync_dart/pubspec.yaml @@ -2,13 +2,14 @@ name: crdt_sync description: >- Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps. Pure Dart, no Flutter dependency. -version: 0.3.0 +version: 0.4.0 publish_to: 'none' environment: sdk: ^3.12.2 dependencies: + crypto: ^3.0.7 http: ^1.6.0 dev_dependencies: diff --git a/crdt_sync_dart/test/firebase_auth_rest_test.dart b/crdt_sync_dart/test/firebase_auth_rest_test.dart new file mode 100644 index 0000000..33f36f9 --- /dev/null +++ b/crdt_sync_dart/test/firebase_auth_rest_test.dart @@ -0,0 +1,248 @@ +import 'dart:convert'; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart' as http_testing; +import 'package:test/test.dart'; + +/// A fixed "now" so token expiry is deterministic rather than wall-clock. +final _now = DateTime.utc(2026, 8, 3, 12); + +http.Response _json(int statusCode, Object body) => + http.Response(jsonEncode(body), statusCode); + +/// Builds a provider whose POSTs return [responses] in call order (each +/// either an [http.Response] or a thrown [http.ClientException]). +FirebaseTokenProvider _provider( + List responses, { + FirebaseCredentialStore? store, + DateTime? now, +}) { + var index = 0; + final mock = http_testing.MockClient((request) async { + final response = responses[index]; + index++; + if (response is http.ClientException) throw response; + return response as http.Response; + }); + return FirebaseTokenProvider( + apiKey: 'fake-api-key', + store: store ?? InMemoryCredentialStore(), + httpClient: mock, + clock: () => now ?? _now, + ); +} + +FirebaseCredentials _credentials({ + String idToken = 'id-1', + String refreshToken = 'refresh-1', + Duration validFor = const Duration(hours: 1), +}) => FirebaseCredentials( + idToken: idToken, + refreshToken: refreshToken, + expiresAt: _now.add(validFor), +); + +void main() { + group('FirebaseCredentials', () { + test('round-trips through JSON', () { + final original = _credentials(); + final restored = FirebaseCredentials.fromJson(original.toJson()); + expect(restored.idToken, original.idToken); + expect(restored.refreshToken, original.refreshToken); + expect(restored.expiresAt, original.expiresAt); + }); + + test('is not expired while comfortably inside its lifetime', () { + expect(_credentials().isExpiredAt(_now), isFalse); + }); + + test('is expired once past expiry', () { + expect( + _credentials(validFor: const Duration(minutes: -1)).isExpiredAt(_now), + isTrue, + ); + }); + + test('is expired inside the refresh skew, before real expiry', () { + // A tick starting now would outlive a token with 2 minutes left, so it + // must be treated as already expired. + expect( + _credentials(validFor: const Duration(minutes: 2)).isExpiredAt(_now), + isTrue, + ); + }); + }); + + group('signIn', () { + test('stores the session returned by identitytoolkit', () async { + final store = InMemoryCredentialStore(); + final provider = _provider([ + _json(200, { + 'idToken': 'id-1', + 'refreshToken': 'refresh-1', + 'expiresIn': '3600', + }), + ], store: store); + + await provider.signIn(email: 'me@example.com', password: 'hunter2'); + + final saved = await store.load(); + expect(saved!.idToken, 'id-1'); + expect(saved.refreshToken, 'refresh-1'); + expect(saved.expiresAt, _now.add(const Duration(hours: 1))); + }); + + test('reports Google\'s reason on a rejected password', () async { + final provider = _provider([ + _json(400, { + 'error': {'message': 'INVALID_LOGIN_CREDENTIALS'}, + }), + ]); + expect( + () => provider.signIn(email: 'me@example.com', password: 'wrong'), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('INVALID_LOGIN_CREDENTIALS'), + ), + ), + ); + }); + + test('reports a string-shaped error body', () async { + final provider = _provider([ + _json(400, {'error': 'BAD_REQUEST'}), + ]); + expect( + () => provider.signIn(email: 'me@example.com', password: 'x'), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('BAD_REQUEST'), + ), + ), + ); + }); + + test('survives a non-JSON error body', () async { + final provider = FirebaseTokenProvider( + apiKey: 'k', + store: InMemoryCredentialStore(), + httpClient: http_testing.MockClient( + (_) async => http.Response('502', 502), + ), + clock: () => _now, + ); + await expectLater( + () => provider.signIn(email: 'a@b.c', password: 'x'), + throwsA(isA()), + ); + }); + + test('turns a network failure into FirebaseAuthError', () async { + final provider = _provider([http.ClientException('offline')]); + expect( + () => provider.signIn(email: 'a@b.c', password: 'x'), + throwsA(isA()), + ); + }); + }); + + group('idToken', () { + test('fails loudly when no session is stored', () { + // Never returns null or a stale token: a sync that quietly stops + // syncing is exactly the failure mode being designed out. + expect( + _provider([]).idToken, + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('not signed in'), + ), + ), + ); + }); + + test('returns the stored token without a network call', () async { + // An empty response list means any HTTP call would throw a range error. + final provider = _provider( + [], + store: InMemoryCredentialStore(_credentials()), + ); + expect(await provider.idToken(), 'id-1'); + }); + + test('refreshes an expired token and persists the rotated refresh token', + () async { + final store = InMemoryCredentialStore( + _credentials(validFor: const Duration(minutes: -5)), + ); + final provider = _provider([ + _json(200, { + 'id_token': 'id-2', + 'refresh_token': 'refresh-2', + 'expires_in': '3600', + }), + ], store: store); + + expect(await provider.idToken(), 'id-2'); + final saved = await store.load(); + expect(saved!.refreshToken, 'refresh-2'); + }); + + test('fails loudly when the refresh token has been revoked', () async { + final provider = _provider([ + _json(400, { + 'error': {'message': 'TOKEN_EXPIRED'}, + }), + ], store: InMemoryCredentialStore( + _credentials(validFor: const Duration(minutes: -5)), + )); + expect( + provider.idToken, + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('TOKEN_EXPIRED'), + ), + ), + ); + }); + }); + + group('session lifecycle', () { + test('hasSession is false before sign-in and true after', () async { + final store = InMemoryCredentialStore(); + final provider = _provider([ + _json(200, { + 'idToken': 'id-1', + 'refreshToken': 'refresh-1', + 'expiresIn': '3600', + }), + ], store: store); + + expect(await provider.hasSession(), isFalse); + await provider.signIn(email: 'a@b.c', password: 'x'); + expect(await provider.hasSession(), isTrue); + }); + + test('signOut clears the store so the next idToken fails', () async { + final provider = _provider( + [], + store: InMemoryCredentialStore(_credentials()), + ); + expect(await provider.idToken(), 'id-1'); + await provider.signOut(); + expect(provider.idToken, throwsA(isA())); + }); + + test('close releases the HTTP client', () { + expect(_provider([]).close, returnsNormally); + }); + }); +} diff --git a/crdt_sync_dart/test/firebase_client_test.dart b/crdt_sync_dart/test/firebase_client_test.dart new file mode 100644 index 0000000..67547bd --- /dev/null +++ b/crdt_sync_dart/test/firebase_client_test.dart @@ -0,0 +1,374 @@ +import 'dart:convert'; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart' as http_testing; +import 'package:test/test.dart'; + +const _databaseUrl = 'https://example-default-rtdb.europe-west1.firebasedatabase.app'; + +http.Response _json(int statusCode, Object? body) => + http.Response(jsonEncode(body), statusCode); + +/// A provider holding a long-lived token, so client tests never hit the auth +/// endpoints -- an HTTP call from the token provider would fail loudly here. +FirebaseTokenProvider _auth() => FirebaseTokenProvider( + apiKey: 'fake-api-key', + store: InMemoryCredentialStore( + FirebaseCredentials( + idToken: 'id-token', + refreshToken: 'refresh-token', + expiresAt: DateTime.utc(2099), + ), + ), + httpClient: http_testing.MockClient( + (_) async => throw StateError('auth must not be contacted'), + ), +); + +/// Records every request the client makes, so tests can assert on the URL +/// (which carries the key escaping and the shallow flag) as well as the +/// decoded result. +late List requests; + +FirebaseRestClient _client( + Object response, { + FirebaseTokenProvider? auth, +}) { + requests = []; + final mock = http_testing.MockClient((request) async { + requests.add(request); + if (response is http.ClientException) throw response; + return response as http.Response; + }); + return FirebaseRestClient( + databaseUrl: _databaseUrl, + auth: auth ?? _auth(), + httpClient: mock, + ); +} + +void main() { + group('key escaping', () { + test('escapes a .json filename into a legal RTDB key', () { + // `.` is illegal in an RTDB key, and the trailing `.json` in a REST URL + // is the format suffix -- so the name cannot be stored verbatim. + expect(encodeKey('log.json'), 'log~2Ejson'); + }); + + test('leaves dot-free device ids untouched', () { + expect(encodeKey('pc'), 'pc'); + expect(encodeKey('32c28cf3-c0eb-423f-8bac-9bda9f158054'), + '32c28cf3-c0eb-423f-8bac-9bda9f158054'); + }); + + test('escapes every character RTDB forbids, plus the escape char', () { + expect(encodeKey(r'a.b$c#d[e]f~g'), r'a~2Eb~24c~23d~5Be~5Df~7Eg'); + }); + + test('round-trips, including a literal escape sequence in the name', () { + // A name that already looks like an escape must survive: `~` is itself + // escaped first, so `log~2Ejson` is not mistaken for `log.json`. + for (final name in [ + 'log.json', + 'plain', + r'a.b$c#d[e]f~g', + 'log~2Ejson', + '~', + '', + ]) { + expect(decodeKey(encodeKey(name)), name, reason: name); + } + }); + + test('escapes each path segment but keeps the separators', () { + expect( + encodePath('todo-sync/notes/abc.json'), + 'todo-sync/notes/abc~2Ejson', + ); + }); + + test('drops empty segments so a root path yields an empty string', () { + expect(encodePath(''), ''); + expect(encodePath('/a//b/'), 'a/b'); + }); + }); + + group('getFileText', () { + test('returns the stored text blob', () async { + final client = _client(_json(200, '{"a":1}')); + expect(await client.getFileText('ns/devices/pc/log.json'), '{"a":1}'); + }); + + test('requests the escaped path with the auth token', () async { + final client = _client(_json(200, '{}')); + await client.getFileText('ns/devices/pc/log.json'); + expect( + requests.single.url.path, + '/ns/devices/pc/log~2Ejson.json', + ); + expect(requests.single.url.queryParameters['auth'], 'id-token'); + }); + + test('returns null for a path nothing has been written to', () async { + // RTDB answers a missing path with literal `null` and a 200, so this is + // benign "no other device has synced yet", not an error. + final client = _client(_json(200, null)); + expect(await client.getFileText('ns/devices/phone/log.json'), isNull); + }); + + test('throws when the value is not a text blob', () async { + final client = _client(_json(200, {'unexpected': 'object'})); + expect( + () => client.getFileText('ns/devices/pc/log.json'), + throwsA(isA()), + ); + }); + + test('raises DatabaseNotFoundError when the rules reject the uid', () async { + final client = _client(_json(401, {'error': 'Permission denied'})); + expect( + () => client.getFileText('ns/devices/pc/log.json'), + throwsA(isA()), + ); + }); + + test('raises DatabaseNotFoundError on 403 too', () async { + final client = _client(_json(403, {'error': 'Permission denied'})); + expect( + () => client.getFileText('ns/devices/pc/log.json'), + throwsA(isA()), + ); + }); + + test('raises a plain FirebaseSyncError on any other non-2xx', () async { + // Spark quota exhaustion lands here. It must surface, never be + // swallowed into a null that reads as "nothing to sync". + final client = _client(_json(429, {'error': 'quota exceeded'})); + expect( + () => client.getFileText('ns/devices/pc/log.json'), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('429'), + ), + ), + ); + }); + + test('turns a network failure into FirebaseSyncError', () async { + final client = _client(http.ClientException('offline')); + expect( + () => client.getFileText('ns/devices/pc/log.json'), + throwsA(isA()), + ); + }); + }); + + group('listDirectory', () { + test('returns decoded keys and asks for a shallow read', () async { + final client = _client(_json(200, {'pc': true, 'phone': true})); + expect(await client.listDirectory('ns/devices'), ['pc', 'phone']); + // shallow=true is what keeps a listing costing bytes instead of the + // whole subtree -- the difference between ~40 MB/month and ~700 MB. + expect(requests.single.url.queryParameters['shallow'], 'true'); + }); + + test('decodes escaped filenames back to what the caller wrote', () async { + // todo-app lists *filenames*, not device directories, so the `.json` + // must come back. + final client = _client(_json(200, {'abc~2Ejson': true})); + expect(await client.listDirectory('todo-sync/notes'), ['abc.json']); + }); + + test('returns empty for a prefix nothing has been written under', () async { + final client = _client(_json(200, null)); + expect(await client.listDirectory('ns/devices'), isEmpty); + }); + + test('raises on a non-2xx response', () async { + final client = _client(_json(500, {'error': 'boom'})); + expect( + () => client.listDirectory('ns/devices'), + throwsA(isA()), + ); + }); + }); + + group('putFileText', () { + test('writes the text as a JSON string leaf', () async { + final client = _client(_json(200, '{"a":1}')); + await client.putFileText( + 'ns/devices/pc/log.json', + '{"a":1}', + message: 'ignored by rtdb', + ); + expect(requests.single.method, 'PUT'); + expect(requests.single.body, jsonEncode('{"a":1}')); + expect(requests.single.url.path, '/ns/devices/pc/log~2Ejson.json'); + }); + + test('raises on a non-2xx response', () async { + final client = _client(_json(400, {'error': 'bad'})); + expect( + () => client.putFileText('ns/devices/pc/log.json', '{}', message: 'm'), + throwsA(isA()), + ); + }); + + test('turns a network failure into FirebaseSyncError', () async { + final client = _client(http.ClientException('offline')); + expect( + () => client.putFileText('ns/devices/pc/log.json', '{}', message: 'm'), + throwsA(isA()), + ); + }); + }); + + group('patchValues', () { + test('PATCHes rather than PUTs, so sibling keys survive', () async { + // A PUT on the shared revs map would wipe every other device's entry, + // after which those devices would look permanently unchanged and never + // be fetched again. + final client = _client(_json(200, {'pc': 'abc'})); + await client.patchValues('ns/revs', {'pc': 'abc'}); + expect(requests.single.method, 'PATCH'); + expect(requests.single.body, jsonEncode({'pc': 'abc'})); + }); + + test('raises on a non-2xx response', () async { + final client = _client(_json(500, {'error': 'boom'})); + expect( + () => client.patchValues('ns/revs', {'pc': 'abc'}), + throwsA(isA()), + ); + }); + + test('turns a network failure into FirebaseSyncError', () async { + final client = _client(http.ClientException('offline')); + expect( + () => client.patchValues('ns/revs', {'pc': 'abc'}), + throwsA(isA()), + ); + }); + }); + + group('getStringMap', () { + test('returns the map with keys decoded', () async { + final client = _client(_json(200, {'pc': 'sha-1', 'phone': 'sha-2'})); + expect(await client.getStringMap('ns/revs'), { + 'pc': 'sha-1', + 'phone': 'sha-2', + }); + }); + + test('degrades to empty when absent', () async { + final client = _client(_json(200, null)); + expect(await client.getStringMap('ns/revs'), isEmpty); + }); + + test('skips non-string entries rather than failing the sync', () async { + // The revs node is an optimisation; a corrupt one must degrade into + // "fetch everything", never into a failed tick. + final client = _client(_json(200, {'pc': 'sha-1', 'phone': 42})); + expect(await client.getStringMap('ns/revs'), {'pc': 'sha-1'}); + }); + + test('raises on a non-2xx response', () async { + final client = _client(_json(500, {'error': 'boom'})); + expect( + () => client.getStringMap('ns/revs'), + throwsA(isA()), + ); + }); + }); + + group('deleteFile', () { + test('sends a DELETE to the escaped path', () async { + final client = _client(_json(200, null)); + await client.deleteFile('ns/devices/pc/log.json'); + expect(requests.single.method, 'DELETE'); + expect(requests.single.url.path, '/ns/devices/pc/log~2Ejson.json'); + }); + + test('raises on a non-2xx response', () async { + final client = _client(_json(500, {'error': 'boom'})); + expect( + () => client.deleteFile('ns/devices/pc/log.json'), + throwsA(isA()), + ); + }); + + test('turns a network failure into FirebaseSyncError', () async { + final client = _client(http.ClientException('offline')); + expect( + () => client.deleteFile('ns/devices/pc/log.json'), + throwsA(isA()), + ); + }); + }); + + group('canAccessRemote', () { + test('is true when the database root reads', () async { + expect(await _client(_json(200, {'ns': true})).canAccessRemote(), isTrue); + }); + + test('is false when the rules reject the token', () async { + expect(await _client(_json(401, {'error': 'denied'})).canAccessRemote(), + isFalse); + }); + + test('is false on a network error', () async { + expect( + await _client(http.ClientException('offline')).canAccessRemote(), + isFalse, + ); + }); + + test('is false when no session is stored, rather than throwing', () async { + // "cannot get a token" is exactly "cannot access the remote", and a + // settings screen's Test-connection button must not blow up. + final client = _client( + _json(200, {}), + auth: FirebaseTokenProvider( + apiKey: 'k', + store: InMemoryCredentialStore(), + httpClient: http_testing.MockClient((_) async => _json(200, {})), + ), + ); + expect(await client.canAccessRemote(), isFalse); + }); + }); + + group('RemoteStore contract', () { + test('FirebaseRestClient is a RemoteStore', () { + expect(_client(_json(200, null)), isA()); + }); + + test('DatabaseNotFoundError is catchable as either error type', () { + final error = DatabaseNotFoundError('gone'); + expect(error, isA()); + expect(error, isA()); + expect(error, isA()); + }); + + test('a trailing slash on the database URL is tolerated', () async { + final client = FirebaseRestClient( + databaseUrl: '$_databaseUrl/', + auth: _auth(), + httpClient: http_testing.MockClient((request) async { + requests.add(request); + return _json(200, 'ok'); + }), + ); + requests = []; + await client.getFileText('ns/x.json'); + expect(requests.single.url.toString(), startsWith('$_databaseUrl/ns/')); + }); + + test('close releases the HTTP client', () { + expect(_client(_json(200, null)).close, returnsNormally); + }); + }); +} diff --git a/crdt_sync_dart/test/github_client_test.dart b/crdt_sync_dart/test/github_client_test.dart index 76c7c74..8770ae1 100644 --- a/crdt_sync_dart/test/github_client_test.dart +++ b/crdt_sync_dart/test/github_client_test.dart @@ -213,6 +213,41 @@ void main() { }); }); + group('canAccessRemote', () { + // The RemoteStore spelling must behave exactly like the legacy name. + test('is true when the repo endpoint returns 2xx', () async { + expect(await _client([_response(200)]).canAccessRemote(), isTrue); + }); + + test('is false when the repo is missing (404)', () async { + expect(await _client([_response(404)]).canAccessRemote(), isFalse); + }); + + test('is false on a network error', () async { + expect(await _client([_networkError]).canAccessRemote(), isFalse); + }); + }); + + group('RemoteStore contract', () { + test('GitHubClient is a RemoteStore', () { + expect(_client([]), isA()); + }); + + test('RepoNotFoundError is catchable as either error type', () { + // Backend-neutral callers catch RemoteNotFoundError; existing GitHub + // callers catch GitHubSyncError. One exception must satisfy both. + final error = RepoNotFoundError('gone'); + expect(error, isA()); + expect(error, isA()); + expect(error, isA()); + }); + + test('errors stringify as their own type', () { + expect(GitHubSyncError('boom').toString(), 'GitHubSyncError: boom'); + expect(RepoNotFoundError('gone').toString(), 'RepoNotFoundError: gone'); + }); + }); + group('deleteFile', () { test('deletes an existing file (resolves its own sha)', () async { final client = _client( diff --git a/crdt_sync_dart/test/mirror_store_test.dart b/crdt_sync_dart/test/mirror_store_test.dart new file mode 100644 index 0000000..60b20a7 --- /dev/null +++ b/crdt_sync_dart/test/mirror_store_test.dart @@ -0,0 +1,361 @@ +/// Tests for the dual-write decorator used during the GitHub -> Firebase +/// cutover. +/// +/// The asymmetry is the whole point and is what these assert: a primary +/// failure must fail the tick, a mirror failure must not, and reads must +/// consult both so that a half-migrated app (one device moved, one not) still +/// converges in both directions. +library; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:test/test.dart'; + +/// A scriptable in-memory [RemoteStore] that can be told to fail. +class _FakeStore implements RemoteStore, BulkMapReader { + _FakeStore({Map? files, this.failing = false}) + : files = files ?? {}; + + final Map files; + bool failing; + final List writes = []; + bool closed = false; + + void _guard(String what) { + if (failing) throw RemoteSyncError('$what failed'); + } + + @override + Future> listDirectory(String path) async { + _guard('list'); + return files.keys + .where((key) => key.startsWith('$path/')) + .map((key) => key.substring(path.length + 1).split('/').first) + .toSet() + .toList(); + } + + @override + Future getFileText(String path) async { + _guard('read'); + return files[path]; + } + + @override + Future putFileText( + String path, + String text, { + required String message, + }) async { + _guard('write'); + writes.add(path); + files[path] = text; + } + + @override + Future> getStringMap(String path) async { + _guard('map read'); + return { + for (final entry in files.entries) + if (entry.key.startsWith('$path/')) + entry.key.substring(path.length + 1): entry.value, + }; + } + + @override + Future deleteFile(String path, {String message = ''}) async { + _guard('delete'); + files.remove(path); + } + + @override + Future canAccessRemote() async => !failing; + + @override + void close() => closed = true; +} + +/// A store with no bulk-map capability, standing in for [GitHubClient]. +class _FakeStoreWithoutBulkRead implements RemoteStore { + _FakeStoreWithoutBulkRead([Map? files]) + : _inner = _FakeStore(files: files); + + final _FakeStore _inner; + + @override + Future> listDirectory(String path) => + _inner.listDirectory(path); + @override + Future getFileText(String path) => _inner.getFileText(path); + @override + Future putFileText(String p, String t, {required String message}) => + _inner.putFileText(p, t, message: message); + @override + Future deleteFile(String path, {String message = ''}) => + _inner.deleteFile(path, message: message); + @override + Future canAccessRemote() => _inner.canAccessRemote(); + @override + void close() => _inner.close(); +} + +({MirrorStore store, _FakeStore primary, _FakeStore mirror, List failures}) +_mirror({ + Map? primaryFiles, + Map? mirrorFiles, + bool primaryFailing = false, + bool mirrorFailing = false, +}) { + final primary = _FakeStore(files: primaryFiles, failing: primaryFailing); + final mirror = _FakeStore(files: mirrorFiles, failing: mirrorFailing); + final failures = []; + return ( + store: MirrorStore( + primary: primary, + mirror: mirror, + onMirrorFailure: (operation, _) => failures.add(operation), + ), + primary: primary, + mirror: mirror, + failures: failures, + ); +} + +void main() { + group('writes', () { + test('go to both backends', () async { + final m = _mirror(); + await m.store.putFileText('ns/pc/log.json', '{}', message: 'm'); + expect(m.primary.writes, ['ns/pc/log.json']); + expect(m.mirror.writes, ['ns/pc/log.json']); + }); + + test('a primary failure fails the tick', () async { + // Fail-closed: the primary is authoritative, so a sync that could not + // write it must not be reported as successful. + final m = _mirror(primaryFailing: true); + await expectLater( + () => m.store.putFileText('ns/pc/log.json', '{}', message: 'm'), + throwsA(isA()), + ); + }); + + test('a mirror failure is loud but does not fail the tick', () async { + final m = _mirror(mirrorFailing: true); + await m.store.putFileText('ns/pc/log.json', '{}', message: 'm'); + expect(m.primary.writes, ['ns/pc/log.json']); + expect(m.failures, ['putFileText ns/pc/log.json']); + }); + + test('deletes behave the same way', () async { + final m = _mirror( + primaryFiles: {'ns/pc/log.json': '{}'}, + mirrorFiles: {'ns/pc/log.json': '{}'}, + mirrorFailing: true, + ); + await m.store.deleteFile('ns/pc/log.json'); + expect(m.primary.files, isEmpty); + expect(m.failures, ['deleteFile ns/pc/log.json']); + }); + + test('a primary delete failure fails the tick', () async { + final m = _mirror(primaryFailing: true); + await expectLater( + () => m.store.deleteFile('ns/pc/log.json'), + throwsA(isA()), + ); + }); + }); + + group('reads consult both backends', () { + test('prefer the primary when it has the file', () async { + final m = _mirror( + primaryFiles: {'ns/pc/log.json': 'from-primary'}, + mirrorFiles: {'ns/pc/log.json': 'from-mirror'}, + ); + expect(await m.store.getFileText('ns/pc/log.json'), 'from-primary'); + }); + + test('fall back to the mirror for an un-migrated device', () async { + // This is why reads are not primary-only: a migrated desktop must + // still see an un-migrated phone's writes, or convergence silently + // becomes one-directional with no error raised. + final m = _mirror(mirrorFiles: {'ns/phone/log.json': 'from-mirror'}); + expect(await m.store.getFileText('ns/phone/log.json'), 'from-mirror'); + }); + + test('return null when neither backend has the file', () async { + final m = _mirror(); + expect(await m.store.getFileText('ns/nobody/log.json'), isNull); + }); + + test('a mirror read failure degrades to the primary answer', () async { + final m = _mirror(mirrorFailing: true); + expect(await m.store.getFileText('ns/pc/log.json'), isNull); + expect(m.failures, ['getFileText ns/pc/log.json']); + }); + + test('a primary read failure fails the tick', () async { + final m = _mirror(primaryFailing: true); + await expectLater( + () => m.store.getFileText('ns/pc/log.json'), + throwsA(isA()), + ); + }); + }); + + group('listDirectory', () { + test('unions devices from both backends', () async { + final m = _mirror( + primaryFiles: {'ns/pc/log.json': '{}'}, + mirrorFiles: {'ns/phone/log.json': '{}'}, + ); + expect(await m.store.listDirectory('ns'), containsAll(['pc', 'phone'])); + }); + + test('does not duplicate a device present in both', () async { + final m = _mirror( + primaryFiles: {'ns/pc/log.json': '{}'}, + mirrorFiles: {'ns/pc/log.json': '{}'}, + ); + expect(await m.store.listDirectory('ns'), ['pc']); + }); + + test('a mirror failure degrades to the primary list', () async { + final m = _mirror( + primaryFiles: {'ns/pc/log.json': '{}'}, + mirrorFailing: true, + ); + expect(await m.store.listDirectory('ns'), ['pc']); + expect(m.failures, ['listDirectory ns']); + }); + + test('a primary failure fails the tick', () async { + final m = _mirror(primaryFailing: true); + await expectLater( + () => m.store.listDirectory('ns'), + throwsA(isA()), + ); + }); + }); + + group('revision maps', () { + test('merge both backends, primary winning on conflict', () async { + final m = _mirror( + primaryFiles: {'ns/revs/pc': 'primary-rev'}, + mirrorFiles: {'ns/revs/pc': 'mirror-rev', 'ns/revs/phone': 'old-rev'}, + ); + expect(await m.store.getStringMap('ns/revs'), { + 'pc': 'primary-rev', + // An un-migrated device publishes revisions only to the mirror; + // without this it would look revision-less and be re-downloaded + // every tick for the whole trial. + 'phone': 'old-rev', + }); + }); + + test('a mirror failure degrades to primary revisions only', () async { + final m = _mirror( + primaryFiles: {'ns/revs/pc': 'primary-rev'}, + mirrorFailing: true, + ); + expect(await m.store.getStringMap('ns/revs'), {'pc': 'primary-rev'}); + expect(m.failures, ['getStringMap ns/revs']); + }); + + test('a backend without bulk reads simply contributes nothing', () async { + final store = MirrorStore( + primary: _FakeStore(files: {'ns/revs/pc': 'primary-rev'}), + mirror: _FakeStoreWithoutBulkRead({'ns/revs/phone': 'ignored'}), + ); + expect(await store.getStringMap('ns/revs'), {'pc': 'primary-rev'}); + }); + + test('is empty when the primary has no bulk reads either', () async { + final store = MirrorStore( + primary: _FakeStoreWithoutBulkRead(), + mirror: _FakeStoreWithoutBulkRead(), + ); + expect(await store.getStringMap('ns/revs'), isEmpty); + }); + }); + + group('lifecycle', () { + test('canAccessRemote reports only the primary', () async { + // A Test-connection button must not report success because the backend + // being retired happens to answer. + final m = _mirror(primaryFailing: true); + expect(await m.store.canAccessRemote(), isFalse); + final healthy = _mirror(mirrorFailing: true); + expect(await healthy.store.canAccessRemote(), isTrue); + }); + + test('close releases both backends', () { + final m = _mirror(); + m.store.close(); + expect(m.primary.closed, isTrue); + expect(m.mirror.closed, isTrue); + }); + + test('a mirror failure with no handler is still swallowed', () async { + final store = MirrorStore( + primary: _FakeStore(), + mirror: _FakeStore(failing: true), + ); + await expectLater( + store.putFileText('ns/pc/log.json', '{}', message: 'm'), + completes, + ); + }); + + test('is itself a RemoteStore, so syncLog accepts it', () { + expect(_mirror().store, isA()); + }); + }); + + group('end to end through syncLog', () { + test('a half-migrated pair still converges both ways', () async { + // desktop is on Firebase+mirror; phone has not migrated and still + // writes only to GitHub. Both must end up with both records. + final github = _FakeStore(); + final firebase = _FakeStore(); + + Log log(String id, String nodeId) => { + id: Record( + id: id, + fields: { + 'v': (id, Hlc(wallTimeMs: 1000, counter: 0, nodeId: nodeId)), + }, + ), + }; + // The library's own serialization, so this exercises the same + // encoding the apps use rather than a test-only one. + const encode = logToJson; + const decode = logFromJson; + + // Un-migrated phone: GitHub only. + await syncLog( + client: github, + deviceId: 'phone', + pathPrefix: 'ns/devices', + localLog: log('from-phone', 'node-phone'), + encode: encode, + decode: decode, + ); + + // Migrated desktop: Firebase primary, GitHub mirror. + final merged = await syncLog( + client: MirrorStore(primary: firebase, mirror: github), + deviceId: 'desktop', + pathPrefix: 'ns/devices', + localLog: log('from-desktop', 'node-desktop'), + encode: encode, + decode: decode, + ); + + expect(merged.keys, containsAll(['from-phone', 'from-desktop'])); + // The desktop's merged result is mirrored back to GitHub, so the + // un-migrated phone sees it on its next tick. + expect(github.files['ns/devices/desktop/log.json'], isNotNull); + }); + }); +} diff --git a/crdt_sync_dart/test/sync_revisions_test.dart b/crdt_sync_dart/test/sync_revisions_test.dart new file mode 100644 index 0000000..4c14f80 --- /dev/null +++ b/crdt_sync_dart/test/sync_revisions_test.dart @@ -0,0 +1,360 @@ +/// Tests for the revision tracking that keeps sync inside the free tier. +/// +/// Two savings, both measured against the GitHub-backed history this +/// replaces: 88.3% of pushes there were byte-identical no-ops, and every tick +/// re-downloaded every peer's whole log regardless of whether it had changed. +/// These tests assert the *request counts*, since that -- not the merge +/// result -- is what the free-tier headroom depends on. +library; + +import 'dart:convert'; + +import 'package:crdt_sync/crdt_sync.dart'; +import 'package:test/test.dart'; + +Log _log(String id, String value, {String nodeId = 'node-a'}) => { + id: Record( + id: id, + fields: { + 'value': (value, Hlc(wallTimeMs: 1000, counter: 0, nodeId: nodeId)), + }, + ), +}; + +String _encode(Log log) => + jsonEncode(log.map((id, record) => MapEntry(id, record.toJson()))); + +Log _decode(String text) => (jsonDecode(text) as Map).map( + (id, data) => MapEntry(id, Record.fromJson(data as Map)), +); + +/// An in-memory [RemoteStore] that also serves revision maps in one read, +/// standing in for [FirebaseRestClient]. Counts every call so tests can +/// assert on traffic rather than on results. +class _FakeRemote implements RemoteStore, BulkMapReader { + _FakeRemote(this.files); + + final Map files; + final List reads = []; + final List writes = []; + final List mapReads = []; + + @override + Future> listDirectory(String path) async => files.keys + .where((key) => key.startsWith('$path/')) + .map((key) => key.substring(path.length + 1).split('/').first) + .toSet() + .toList(); + + @override + Future getFileText(String path) async { + reads.add(path); + return files[path]; + } + + @override + Future putFileText( + String path, + String text, { + required String message, + }) async { + writes.add(path); + files[path] = text; + } + + @override + Future> getStringMap(String path) async { + mapReads.add(path); + return { + for (final entry in files.entries) + if (entry.key.startsWith('$path/')) + entry.key.substring(path.length + 1): entry.value, + }; + } + + @override + Future deleteFile(String path, {String message = ''}) async => + files.remove(path); + + @override + Future canAccessRemote() async => true; + + @override + void close() {} +} + +/// A [RemoteStore] with no bulk-map capability, standing in for +/// [GitHubClient]: revision tracking must degrade, not break. +/// +/// Delegates rather than extends, because a subclass of [_FakeRemote] would +/// inherit `BulkMapReader` -- the very capability this fake exists to lack. +class _FakeRemoteWithoutBulkRead implements RemoteStore { + _FakeRemoteWithoutBulkRead(Map files) + : _inner = _FakeRemote(files); + + final _FakeRemote _inner; + + Map get files => _inner.files; + List get reads => _inner.reads; + List get writes => _inner.writes; + + @override + Future> listDirectory(String path) => + _inner.listDirectory(path); + + @override + Future getFileText(String path) => _inner.getFileText(path); + + @override + Future putFileText( + String path, + String text, { + required String message, + }) => _inner.putFileText(path, text, message: message); + + @override + Future deleteFile(String path, {String message = ''}) => + _inner.deleteFile(path, message: message); + + @override + Future canAccessRemote() => _inner.canAccessRemote(); + + @override + void close() => _inner.close(); +} + +Future _tick( + RemoteStore remote, + SyncStateStore? store, { + Log local = const {}, + String deviceId = 'pc', +}) => syncLog( + client: remote, + deviceId: deviceId, + pathPrefix: 'ns/devices', + localLog: local, + encode: _encode, + decode: _decode, + stateStore: store, +); + +void main() { + group('revisionOf', () { + test('is stable for identical content', () { + expect(revisionOf('{"a":1}'), revisionOf('{"a":1}')); + }); + + test('differs for different content', () { + expect(revisionOf('{"a":1}'), isNot(revisionOf('{"a":2}'))); + }); + }); + + group('defaultRevsPath', () { + test('is a sibling of the device directory', () { + expect(defaultRevsPath('diet-guard-sync/devices'), 'diet-guard-sync/revs'); + expect(defaultRevsPath('todo-sync/notes'), 'todo-sync/revs'); + }); + + test('falls back to a child when the prefix has no parent', () { + expect(defaultRevsPath('ns'), 'ns/revs'); + }); + }); + + group('SyncState', () { + test('round-trips through JSON', () { + const state = SyncState(pushedRev: 'abc', peerRevs: {'phone': 'def'}); + final restored = SyncState.fromJson(state.toJson()); + expect(restored.pushedRev, 'abc'); + expect(restored.peerRevs, {'phone': 'def'}); + }); + + test('tolerates a missing or malformed peer map', () { + expect(SyncState.fromJson({}).peerRevs, isEmpty); + expect( + SyncState.fromJson({ + 'peer_revs': {'phone': 42}, + }).peerRevs, + isEmpty, + ); + }); + }); + + group('no-op push suppression', () { + test('pushes on the first tick and publishes a revision', () async { + final remote = _FakeRemote({}); + final store = InMemorySyncStateStore(); + await _tick(remote, store, local: _log('a', '1')); + expect(remote.writes, [ + 'ns/devices/pc/log.json', + 'ns/revs/pc', + ]); + }); + + test('a second unchanged tick writes nothing at all', () async { + final remote = _FakeRemote({}); + final store = InMemorySyncStateStore(); + final local = _log('a', '1'); + await _tick(remote, store, local: local); + remote.writes.clear(); + + await _tick(remote, store, local: local); + // This is the 88.3% of the old history that was pure waste. + expect(remote.writes, isEmpty); + }); + + test('a changed log pushes again', () async { + final remote = _FakeRemote({}); + final store = InMemorySyncStateStore(); + await _tick(remote, store, local: _log('a', '1')); + remote.writes.clear(); + + await _tick(remote, store, local: _log('a', '2')); + expect(remote.writes, contains('ns/devices/pc/log.json')); + }); + + test('without a state store every tick pushes, as it always did', () async { + final remote = _FakeRemote({}); + final local = _log('a', '1'); + await _tick(remote, null, local: local); + await _tick(remote, null, local: local); + expect( + remote.writes, + ['ns/devices/pc/log.json', 'ns/devices/pc/log.json'], + ); + // No state store means no revision publishing either. + expect(remote.writes.where((w) => w.startsWith('ns/revs')), isEmpty); + }); + }); + + group('peer download suppression', () { + test('downloads a peer whose revision it has never seen', () async { + final peer = _encode(_log('b', 'from-phone', nodeId: 'node-b')); + final remote = _FakeRemote({ + 'ns/devices/phone/log.json': peer, + 'ns/revs/phone': revisionOf(peer), + }); + final merged = await _tick(remote, InMemorySyncStateStore()); + expect(remote.reads, contains('ns/devices/phone/log.json')); + expect(merged.keys, contains('b')); + }); + + test('skips the download when the peer revision is unchanged', () async { + final peer = _encode(_log('b', 'from-phone', nodeId: 'node-b')); + final remote = _FakeRemote({ + 'ns/devices/phone/log.json': peer, + 'ns/revs/phone': revisionOf(peer), + }); + final store = InMemorySyncStateStore(); + final first = await _tick(remote, store); + remote.reads.clear(); + + // The peer's records are already in the local log from tick one, so + // re-downloading them is pure waste -- this is the ~700 MB/month that + // the revision check removes. + await _tick(remote, store, local: first); + expect(remote.reads, isEmpty); + }); + + test('downloads again once the peer publishes a new revision', () async { + final peerV1 = _encode(_log('b', 'v1', nodeId: 'node-b')); + final remote = _FakeRemote({ + 'ns/devices/phone/log.json': peerV1, + 'ns/revs/phone': revisionOf(peerV1), + }); + final store = InMemorySyncStateStore(); + final first = await _tick(remote, store); + remote.reads.clear(); + + final peerV2 = _encode({ + ..._decode(peerV1), + ..._log('c', 'v2', nodeId: 'node-b'), + }); + remote.files['ns/devices/phone/log.json'] = peerV2; + remote.files['ns/revs/phone'] = revisionOf(peerV2); + + final second = await _tick(remote, store, local: first); + expect(remote.reads, contains('ns/devices/phone/log.json')); + expect(second.keys, containsAll(['b', 'c'])); + }); + + test('re-downloads a peer whose push was corrupt', () async { + // A failed decode must not be remembered as seen, or the corruption + // would be permanent. + final remote = _FakeRemote({ + 'ns/devices/phone/log.json': 'not json at all', + 'ns/revs/phone': revisionOf('not json at all'), + }); + final store = InMemorySyncStateStore(); + await _tick(remote, store); + remote.reads.clear(); + + await _tick(remote, store); + expect(remote.reads, contains('ns/devices/phone/log.json')); + }); + + test('downloads when the peer has published no revision yet', () async { + // A device still running the pre-migration code publishes a log but no + // revision; it must not be silently ignored. + final peer = _encode(_log('b', 'from-phone', nodeId: 'node-b')); + final remote = _FakeRemote({'ns/devices/phone/log.json': peer}); + final store = InMemorySyncStateStore(); + final merged = await _tick(remote, store); + expect(merged.keys, contains('b')); + }); + + test('skips a peer that has nothing pushed yet', () async { + final remote = _FakeRemote({'ns/devices/phone/other.txt': 'x'}); + final merged = await _tick(remote, InMemorySyncStateStore()); + expect(merged, isEmpty); + }); + + test('never reads its own device back', () async { + final remote = _FakeRemote({ + 'ns/devices/pc/log.json': _encode(_log('a', '1')), + 'ns/revs/pc': 'whatever', + }); + await _tick(remote, InMemorySyncStateStore()); + expect(remote.reads, isEmpty); + }); + }); + + group('backends without bulk-map reads', () { + test('still sync correctly, just without the saving', () async { + // GitHubClient has no BulkMapReader, so revision lookup degrades to + // "fetch everything" -- correctness must not depend on the optimisation. + final peer = _encode(_log('b', 'from-phone', nodeId: 'node-b')); + final remote = _FakeRemoteWithoutBulkRead({ + 'ns/devices/phone/log.json': peer, + }); + final store = InMemorySyncStateStore(); + final first = await _tick(remote, store); + expect(first.keys, contains('b')); + + remote.reads.clear(); + await _tick(remote, store, local: first); + expect(remote.reads, contains('ns/devices/phone/log.json')); + }); + }); + + group('revision publishing order', () { + test('publishes the log before its revision', () async { + // Reversed, a peer would cache "seen rev X" against a log it never + // received, and skip it forever. + final remote = _FakeRemote({}); + await _tick(remote, InMemorySyncStateStore(), local: _log('a', '1')); + expect( + remote.writes.indexOf('ns/devices/pc/log.json'), + lessThan(remote.writes.indexOf('ns/revs/pc')), + ); + }); + + test('each device writes only its own revision key', () async { + // Per-device keys rather than one shared map: a whole-map write would + // erase every other device's entry, after which those peers would look + // permanently unchanged and never be fetched again. + final remote = _FakeRemote({'ns/revs/phone': 'peer-rev'}); + await _tick(remote, InMemorySyncStateStore(), local: _log('a', '1')); + expect(remote.files['ns/revs/phone'], 'peer-rev'); + }); + }); +} diff --git a/crdt_sync_dart/tool/interop_check.dart b/crdt_sync_dart/tool/interop_check.dart new file mode 100644 index 0000000..e20a74d --- /dev/null +++ b/crdt_sync_dart/tool/interop_check.dart @@ -0,0 +1,71 @@ +/// Live cross-language check: reads what the Python client wrote, merges a +/// Dart-side record, and confirms both survive. The two implementations must +/// agree byte-for-byte or a phone on Dart would not see a PC on Python. +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:crdt_sync/crdt_sync.dart'; + +/// Environment-specific values (project, account, database URL) live here, +/// mode 0600 -- never in source, because this repository is public. +final _configDir = '${Platform.environment['HOME']}/.config/crdt-sync'; + +Log _decode(String t) => (jsonDecode(t) as Map) + .map((k, v) => MapEntry(k, Record.fromJson(v as Map))); +String _encode(Log l) => + jsonEncode(l.map((k, v) => MapEntry(k, v.toJson()))); + +Future main(List args) async { + final cfg = jsonDecode( + File('$_configDir/firebase.json').readAsStringSync(), + ) as Map; + final auth = FirebaseTokenProvider( + apiKey: cfg['apiKey'] as String, + store: InMemoryCredentialStore(), + ); + await auth.signIn( + email: cfg['email'] as String, + password: File('$_configDir/password').readAsStringSync(), + ); + final client = FirebaseRestClient( + databaseUrl: cfg['databaseUrl'] as String, + auth: auth, + ); + + final merged = await syncLog( + client: client, + deviceId: 'dartdev', + pathPrefix: '_interop/devices', + localLog: { + 'dart-rec': Record( + id: 'dart-rec', + fields: { + 'value': ( + 'written-by-dart', + Hlc(wallTimeMs: 1000, counter: 0, nodeId: 'node-dart'), + ), + }, + ), + }, + encode: _encode, + decode: _decode, + stateStore: InMemorySyncStateStore(), + ); + + final ok = merged.containsKey('py-rec') && merged.containsKey('dart-rec'); + stdout.writeln('dart sees records: ${merged.keys.toList()}'); + stdout.writeln( + 'python record value: ${merged['py-rec']?.fields['value']?.$1}', + ); + stdout.writeln('revs: ${await client.getStringMap('_interop/revs')}'); + + // Clean up both devices' data. + for (final d in ['pydev', 'dartdev']) { + await client.deleteFile('_interop/devices/$d/log.json'); + await client.deleteFile('_interop/revs/$d'); + } + stdout.writeln(ok ? '\nINTEROP OK' : '\nINTEROP FAILED'); + exit(ok ? 0 : 1); +} diff --git a/crdt_sync_dart/tool/live_smoke_test.dart b/crdt_sync_dart/tool/live_smoke_test.dart new file mode 100644 index 0000000..bbbde7b --- /dev/null +++ b/crdt_sync_dart/tool/live_smoke_test.dart @@ -0,0 +1,166 @@ +/// End-to-end smoke test against the real Firebase Realtime Database. +/// +/// Mocked tests prove the client's branches; this proves the thing actually +/// works against Google's servers -- key escaping, auth, shallow listing, and +/// the revision-based traffic savings that the free-tier headroom depends on. +/// +/// Usage: +/// `dart run tool/live_smoke_test.dart ` +/// +/// Writes only under `_smoketest/`, and deletes what it wrote on the way out. +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:crdt_sync/crdt_sync.dart'; + +/// Environment-specific values live in a local 0600 config file rather than +/// in source, because this repository is public. +final _configDir = '${Platform.environment['HOME']}/.config/crdt-sync'; +const _prefix = '_smoketest/devices'; + +var _failures = 0; + +void _check(String what, bool ok, [String detail = '']) { + stdout.writeln('${ok ? " ok " : " FAIL"} $what${detail.isEmpty ? '' : ' ($detail)'}'); + if (!ok) _failures++; +} + +Log _log(String id, String value, String nodeId) => { + id: Record( + id: id, + fields: { + 'value': (value, Hlc(wallTimeMs: 1000, counter: 0, nodeId: nodeId)), + }, + ), +}; + +String _encode(Log log) => + jsonEncode(log.map((id, record) => MapEntry(id, record.toJson()))); + +Log _decode(String text) => (jsonDecode(text) as Map).map( + (id, data) => MapEntry(id, Record.fromJson(data as Map)), +); + +Future main(List args) async { + if (args.length != 3) { + stderr.writeln( + 'usage: dart run tool/live_smoke_test.dart ', + ); + exit(2); + } + final [apiKey, email, password] = args; + final cfg = + jsonDecode(File('$_configDir/firebase.json').readAsStringSync()) + as Map; + + final auth = FirebaseTokenProvider( + apiKey: apiKey, + store: InMemoryCredentialStore(), + ); + final client = FirebaseRestClient( + databaseUrl: cfg['databaseUrl'] as String, + auth: auth, + ); + + stdout.writeln('\n== auth =='); + _check('canAccessRemote is false before sign-in', !await auth.hasSession()); + await auth.signIn(email: email, password: password); + _check('signIn stored a session', await auth.hasSession()); + _check('canAccessRemote is true after sign-in', await client.canAccessRemote()); + + stdout.writeln('\n== round-trip and key escaping =='); + const path = '$_prefix/pc/log.json'; + await client.putFileText(path, '{"hello":"world"}', message: 'smoke'); + final read = await client.getFileText(path); + _check('a .json filename round-trips byte-for-byte', + read == '{"hello":"world"}', read ?? 'null'); + _check('missing path reads as null', + await client.getFileText('$_prefix/nope/log.json') == null); + final listed = await client.listDirectory(_prefix); + _check('listDirectory sees the device', listed.contains('pc'), '$listed'); + + stdout.writeln('\n== sync: first tick pushes =='); + final storePc = InMemorySyncStateStore(); + await client.deleteFile(path); + var pcLog = await syncLog( + client: client, + deviceId: 'pc', + pathPrefix: _prefix, + localLog: _log('a', 'from-pc', 'node-pc'), + encode: _encode, + decode: _decode, + stateStore: storePc, + ); + final pushed = await client.getFileText(path); + _check('the log landed in the database', pushed != null); + final revs = await client.getStringMap('_smoketest/revs'); + _check('a revision was published for this device', + revs['pc'] == revisionOf(_encode(pcLog)), '${revs.keys}'); + + stdout.writeln('\n== sync: an unchanged tick writes nothing =='); + final before = await client.getStringMap('_smoketest/revs'); + pcLog = await syncLog( + client: client, + deviceId: 'pc', + pathPrefix: _prefix, + localLog: pcLog, + encode: _encode, + decode: _decode, + stateStore: storePc, + ); + _check('revision unchanged, so nothing was re-uploaded', + (await client.getStringMap('_smoketest/revs'))['pc'] == before['pc']); + + stdout.writeln('\n== sync: two devices converge =='); + final storePhone = InMemorySyncStateStore(); + final phoneLog = await syncLog( + client: client, + deviceId: 'phone', + pathPrefix: _prefix, + localLog: _log('b', 'from-phone', 'node-phone'), + encode: _encode, + decode: _decode, + stateStore: storePhone, + ); + _check('phone merged the pc record', phoneLog.containsKey('a')); + final pcAfter = await syncLog( + client: client, + deviceId: 'pc', + pathPrefix: _prefix, + localLog: pcLog, + encode: _encode, + decode: _decode, + stateStore: storePc, + ); + _check('pc merged the phone record', pcAfter.containsKey('b')); + _check('each device wrote only its own revision key', + (await client.getStringMap('_smoketest/revs')).length == 2); + + stdout.writeln('\n== fail-closed =='); + await auth.signOut(); + var threw = false; + try { + await client.getFileText(path); + } on RemoteSyncError { + threw = true; + } + _check('a revoked session fails loudly rather than returning null', threw); + + stdout.writeln('\n== cleanup =='); + await auth.signIn(email: email, password: password); + for (final device in ['pc', 'phone']) { + await client.deleteFile('$_prefix/$device/log.json'); + await client.deleteFile('_smoketest/revs/$device'); + } + final leftover = await client.listDirectory('_smoketest'); + _check('smoke-test data removed', leftover.isEmpty, '$leftover'); + + client.close(); + auth.close(); + stdout.writeln( + _failures == 0 ? '\nALL LIVE CHECKS PASSED' : '\n$_failures LIVE CHECK(S) FAILED', + ); + exit(_failures == 0 ? 0 : 1); +} diff --git a/firebase/README.md b/firebase/README.md new file mode 100644 index 0000000..71436d8 --- /dev/null +++ b/firebase/README.md @@ -0,0 +1,72 @@ +# firebase/ + +Deployable configuration for the Firebase project backing `crdt_sync` / +`crdt_sync_dart` sync. Language-neutral, so it lives here rather than under +either library. + +**This repository is public.** Nothing here names a concrete project, database +URL, account or uid. Those live in a local, `0600` config file — see +[Local configuration](#local-configuration). + +## Why Realtime Database, over REST + +* **No SDK.** Official FlutterFire has no Linux desktop support, and one + caller (`wake_alarm`'s PC side) is a dependency-free systemd job. The REST + endpoints are plain HTTPS and behave identically on Linux, Android and + headless. +* **No per-operation quota on the Spark (no-cost) plan.** RTDB bills only + storage and bandwidth, so a misbehaving sync loop cannot exhaust a daily + budget and silently stop working mid-day. Firestore's 20k-writes/day cap + would be a second way to hit a wall, for querying this workload never does. +* **Path model matches the existing layout** — `//` — + so no data remodelling. + +Deliberately **not** used: Cloud Storage (requires a linked billing account +since 2026‑02‑03) and Cloud Functions. Avoiding both keeps the project +structurally incapable of generating a bill. + +## `database.rules.json` + +Scopes the whole database to a single Firebase Auth uid. + +`auth != null` alone would **not** be safe: email/password sign-up is open +through the identitytoolkit REST API to anyone holding the Web API key, and +that key ships inside the Android APKs. Pinning the uid means a stranger who +registers an account still reads and writes nothing. + +To deploy, substitute the real uid (Firebase console → Authentication → Users +→ User UID) and paste into **Realtime Database → Rules → Publish**. + +Verify the rules are live — an unauthenticated request must be refused: + +```bash +curl -s "$DATABASE_URL/.json" +# {"error" : "Permission denied"} <- expected +``` + +## Local configuration + +Everything environment-specific lives in `~/.config/crdt-sync/`, mode `0600`, +outside version control: + +| File | Contents | +|---|---| +| `firebase.json` | `apiKey`, `databaseUrl`, `projectId`, `uid`, `email` | +| `password` | the sync account's password, no trailing newline | +| `database.rules.json` | the deployed rules, with the real uid filled in | + +The Web API key is **not** a secret — it is a public project identifier that +ships inside client apps. The password and refresh tokens are the real +secrets. + +## Credentials at runtime + +No service-account key is used anywhere: it would bypass the security rules +entirely and is trivially extractable from an installed APK. Devices sign in +as the single user and hold a refresh token: + +* Android — `flutter_secure_storage` +* Linux — `~/.config//firebase.json`, mode `0600` + +ID tokens last about an hour, so the refresh exchange is mandatory rather than +optional — `diet_guard`'s PC timer alone runs 96 times a day. diff --git a/firebase/database.rules.json b/firebase/database.rules.json new file mode 100644 index 0000000..1174fd1 --- /dev/null +++ b/firebase/database.rules.json @@ -0,0 +1,6 @@ +{ + "rules": { + ".read": "auth != null && auth.uid === 'YOUR_FIREBASE_UID'", + ".write": "auth != null && auth.uid === 'YOUR_FIREBASE_UID'" + } +}