crdt_sync 0.5.0: the shared config and the cutover seam

Everything an app needs to move from GitHub to Firebase without each one
reinventing the wiring.

Python:
- _config.py: FirebaseConfig.load() + firebase_client_for(app) +
  mirror_client_for(app, github). One config file, read in one place.
  Credential caches stay per-app (~/.config/<app>/firebase_auth.json) --
  one file written by several processes is a corruption waiting to happen.
- tool/preflight_firebase.py: validates the config before any data moves.
  Catches a wrong-region databaseUrl (which answers 404 with a correctUrl
  body rather than an error, so it would otherwise pass every check and
  break mid-migration) and confirms the uid in the ID token matches the uid
  the rules pin -- sign_in() returns None and discards localId, so the uid
  is decoded from the JWT.
- tool/interop_seed.py: the missing Python half of the cross-language
  check. interop_check.dart asserts it can see a record written by Python,
  but nothing ever wrote one, so it could only ever fail.

Dart:
- firebase_config.dart: the same config shape, kept free of dart:io so it
  works on web and Android, where the file does not exist at all and
  credentials belong in flutter_secure_storage.
- PersistedSyncStateStore: the revision cache, persisted. Without it a
  phone forgets every peer revision on each cold start and re-downloads
  everything -- the exact traffic the revision scheme exists to avoid, and
  what the free-tier headroom depends on.

Verified live against the real database: preflight passes all five checks,
the Dart smoke test passes end to end, and Python <-> Dart interop is OK.

Python 226 tests at 100% branch coverage; Dart 218 tests, both new files
at 100% line coverage.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SJ9sxEw3ymzXNjL7SHNpJc
This commit is contained in:
Krzysztof Rudnicki 2026-08-04 22:59:48 +02:00
parent a89f152564
commit dd6e6d5571
12 changed files with 1332 additions and 18 deletions

View File

@ -2,6 +2,16 @@
from __future__ import annotations
from crdt_sync._config import (
CONFIG_DIR,
CONFIG_FILE,
PASSWORD_FILE,
ConfigError,
FirebaseConfig,
credential_store_for,
firebase_client_for,
mirror_client_for,
)
from crdt_sync._firebase import (
DatabaseNotFoundError,
FirebaseSyncClient,
@ -33,12 +43,17 @@ from crdt_sync._sync import (
)
__all__ = [
"CONFIG_DIR",
"CONFIG_FILE",
"PASSWORD_FILE",
"ConfigError",
"CredentialStore",
"DatabaseNotFoundError",
"Field",
"FileCredentialStore",
"FileSyncStateStore",
"FirebaseAuthError",
"FirebaseConfig",
"FirebaseCredentials",
"FirebaseSyncClient",
"FirebaseSyncError",
@ -57,12 +72,15 @@ __all__ = [
"RepoNotFoundError",
"SyncState",
"SyncStateStore",
"credential_store_for",
"default_revs_path",
"dump_log",
"firebase_client_for",
"load_log",
"merge_field",
"merge_logs",
"merge_record",
"mirror_client_for",
"read_log",
"revision_of",
"sync_log",

View File

@ -0,0 +1,192 @@
"""The one place every app learns how to reach Firebase.
Before this module each caller re-read ``~/.config/crdt-sync/firebase.json``
by hand, so a change to the schema meant editing every consumer, and each
consumer got to invent its own half of the credential-cache convention. The
apps share one Firebase project and one account, so they should share one
loader.
Two things live here:
* :class:`FirebaseConfig` -- the parsed, validated config file.
* :func:`firebase_client_for` -- the whole "config file to usable client"
path, including the per-app credential cache that keeps a fresh process
from re-authenticating. ``wake_alarm``'s PC side runs every minute and
``diet_guard``'s every 15, so caching the refresh token is what makes the
sign-in cost negligible rather than constant.
Deliberately *not* here: any choice about which backend an app uses. That
stays a constructor change at the call site, per :mod:`crdt_sync._remote`.
"""
from __future__ import annotations
from dataclasses import dataclass
import json
from pathlib import Path
from typing import TYPE_CHECKING
from crdt_sync._firebase import FirebaseSyncClient
from crdt_sync._firebase_auth import FileCredentialStore, FirebaseTokenProvider
from crdt_sync._mirror import MirrorSyncClient
if TYPE_CHECKING:
from crdt_sync._remote import RemoteStore
CONFIG_DIR = Path.home() / ".config" / "crdt-sync"
CONFIG_FILE = CONFIG_DIR / "firebase.json"
PASSWORD_FILE = CONFIG_DIR / "password"
# The scaffold ships "_comment_*" keys explaining where each value comes
# from. They are documentation for a human filling the file in, not config.
_COMMENT_PREFIX = "_comment"
class ConfigError(Exception):
"""The shared Firebase config is missing, malformed or unfilled."""
@dataclass(frozen=True)
class FirebaseConfig:
"""Everything needed to reach the shared Firebase project.
``api_key`` is not a secret -- it ships inside the Android APKs, and the
security rules are what actually protect the data. The password is.
"""
api_key: str
database_url: str
project_id: str
uid: str
email: str
@classmethod
def load(cls, path: Path | None = None) -> FirebaseConfig:
"""Return the config at ``path``, defaulting to :data:`CONFIG_FILE`.
Raises:
ConfigError: If the file is absent, unparsable, missing a key, or
still holding a scaffold placeholder -- each reported with the
specific field at fault, because the alternative is a
confusing auth failure much later.
"""
source = path if path is not None else CONFIG_FILE
try:
raw = json.loads(source.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
msg = f"{source} does not exist; see utils/firebase/README.md"
raise ConfigError(msg) from exc
except OSError as exc:
msg = f"{source} could not be read: {exc}"
raise ConfigError(msg) from exc
except ValueError as exc:
msg = f"{source} is not valid JSON: {exc}"
raise ConfigError(msg) from exc
if not isinstance(raw, dict):
msg = f"{source} must contain a JSON object"
raise ConfigError(msg)
data = {k: v for k, v in raw.items() if not k.startswith(_COMMENT_PREFIX)}
fields = {
"api_key": "apiKey",
"database_url": "databaseUrl",
"project_id": "projectId",
"uid": "uid",
"email": "email",
}
missing = [key for key in fields.values() if not str(data.get(key, "")).strip()]
if missing:
msg = f"{source} is missing or has empty: {', '.join(missing)}"
raise ConfigError(msg)
unfilled = [key for key in fields.values() if "PASTE_" in str(data[key])]
if unfilled:
msg = f"{source} still holds the placeholder for: {', '.join(unfilled)}"
raise ConfigError(msg)
return cls(**{attr: str(data[key]) for attr, key in fields.items()})
def read_password(self, path: Path | None = None) -> str:
"""Return the sync account's password.
Stripped, because a text editor that appends a trailing newline would
otherwise turn into an authentication failure with no visible cause.
"""
source = path if path is not None else PASSWORD_FILE
try:
password = source.read_text(encoding="utf-8").strip()
except OSError as exc:
msg = f"{source} could not be read: {exc}"
raise ConfigError(msg) from exc
if not password or "PASTE_" in password:
msg = f"{source} is empty or still holds the scaffold placeholder"
raise ConfigError(msg)
return password
def credential_store_for(app_name: str) -> FileCredentialStore:
"""Return the credential cache for ``app_name``.
Per app rather than shared: two apps signing in as the same account still
refresh tokens on their own schedules, and a single file written by
several processes at once is a corruption waiting to happen.
"""
return FileCredentialStore(
Path.home() / ".config" / app_name / "firebase_auth.json"
)
def firebase_client_for(
app_name: str,
*,
config: FirebaseConfig | None = None,
) -> FirebaseSyncClient:
"""Return a signed-in client for ``app_name``.
Reuses the cached refresh token when there is one and signs in with the
password only when there is not, so the common path costs no
authentication round trip at all.
Args:
app_name: Names the credential cache, e.g. ``"diet_guard"``.
config: Overrides the on-disk config; for tests.
Raises:
ConfigError: If the shared config is unusable.
FirebaseAuthError: If the credentials are rejected.
"""
resolved = config if config is not None else FirebaseConfig.load()
auth = FirebaseTokenProvider(resolved.api_key, credential_store_for(app_name))
if not auth.has_session():
auth.sign_in(resolved.email, resolved.read_password())
return FirebaseSyncClient(resolved.database_url, auth)
def mirror_client_for(
app_name: str,
github_client: RemoteStore,
*,
config: FirebaseConfig | None = None,
) -> MirrorSyncClient:
"""Return a Firebase-primary client that still mirrors to ``github_client``.
This is what an app uses *during* the cutover. Firebase is authoritative,
so its failures fail the tick; GitHub is kept in step but never allowed
to. Reads union both, which is what lets devices cut over one at a time
without a migrated PC losing sight of an un-migrated phone.
Rolling back is deleting the call: pass the GitHub client straight to
:func:`crdt_sync.sync_log` again. Retiring the mirror is swapping this for
:func:`firebase_client_for`. Both are one-line changes at the call site,
which is the whole point of the :class:`~crdt_sync.RemoteStore` seam.
Args:
app_name: Names the credential cache, e.g. ``"diet_guard"``.
github_client: The existing GitHub client, kept as the fallback.
config: Overrides the on-disk config; for tests.
"""
return MirrorSyncClient(
primary=firebase_client_for(app_name, config=config),
mirror=github_client,
)

View File

@ -0,0 +1,285 @@
"""Tests for the shared Firebase configuration loader.
Every failure mode here is one that would otherwise surface as an
authentication error long after the real mistake (a placeholder left in, a
newline a text editor appended), so each is asserted to name the field at
fault rather than merely to raise.
"""
from __future__ import annotations
import json
from typing import TYPE_CHECKING
import pytest
from crdt_sync import (
ConfigError,
FirebaseConfig,
FirebaseSyncClient,
credential_store_for,
mirror_client_for,
)
from crdt_sync._config import firebase_client_for
if TYPE_CHECKING:
from pathlib import Path
_VALID = {
"apiKey": "AIzaSyExample",
"databaseUrl": "https://kuhy-syncs-default-rtdb.europe-west1.firebasedatabase.app",
"projectId": "kuhy-syncs",
"uid": "OvA2REQyLIhAHOEjzwS1o877rgG3",
"email": "sync@example.com",
}
class _StubRemote:
"""A do-nothing :class:`~crdt_sync.RemoteStore`, standing in for GitHub.
A real implementation rather than a bare ``object()`` so the type checker
accepts it without a suppression.
"""
def list_directory(self, path: str) -> list[str]:
return []
def get_file_text(self, path: str) -> str | None:
return None
def put_file_text(self, path: str, text: str, *, message: str) -> None:
"""Accept and discard a write."""
def delete_file(self, path: str, *, message: str = "") -> None:
"""Accept and discard a delete."""
def can_access_remote(self) -> bool:
return True
def _write(path: Path, data: object) -> Path:
"""Write ``data`` as JSON to ``path`` and return it."""
path.write_text(json.dumps(data), encoding="utf-8")
return path
def test_loads_a_valid_config(tmp_path: Path) -> None:
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
assert config.api_key == "AIzaSyExample"
assert config.project_id == "kuhy-syncs"
assert config.uid == "OvA2REQyLIhAHOEjzwS1o877rgG3"
assert config.email == "sync@example.com"
assert config.database_url.endswith("europe-west1.firebasedatabase.app")
def test_ignores_the_scaffold_comment_keys(tmp_path: Path) -> None:
"""The shipped scaffold explains each field inline; that is not config."""
annotated = {**_VALID, "_comment_apiKey": "where to find it", "_comment": "x"}
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", annotated))
assert config.api_key == "AIzaSyExample"
def test_reports_a_missing_file(tmp_path: Path) -> None:
with pytest.raises(ConfigError, match="does not exist"):
FirebaseConfig.load(tmp_path / "absent.json")
def test_reports_unreadable_json(tmp_path: Path) -> None:
path = tmp_path / "firebase.json"
path.write_text("{not json", encoding="utf-8")
with pytest.raises(ConfigError, match="not valid JSON"):
FirebaseConfig.load(path)
def test_reports_a_non_object_document(tmp_path: Path) -> None:
with pytest.raises(ConfigError, match="must contain a JSON object"):
FirebaseConfig.load(_write(tmp_path / "firebase.json", ["a list"]))
def test_reports_an_unreadable_path(tmp_path: Path) -> None:
"""A directory where a file belongs is an OSError, not a missing file."""
(tmp_path / "firebase.json").mkdir()
with pytest.raises(ConfigError, match="could not be read"):
FirebaseConfig.load(tmp_path / "firebase.json")
@pytest.mark.parametrize("field", sorted(_VALID))
def test_names_a_missing_field(tmp_path: Path, field: str) -> None:
incomplete = {k: v for k, v in _VALID.items() if k != field}
with pytest.raises(ConfigError, match=field):
FirebaseConfig.load(_write(tmp_path / "firebase.json", incomplete))
def test_names_an_empty_field(tmp_path: Path) -> None:
with pytest.raises(ConfigError, match="uid"):
FirebaseConfig.load(_write(tmp_path / "firebase.json", {**_VALID, "uid": " "}))
def test_names_an_unfilled_placeholder(tmp_path: Path) -> None:
"""The exact state the scaffold ships in must not load silently."""
unfilled = {**_VALID, "apiKey": "PASTE_WEB_API_KEY_HERE"}
with pytest.raises(ConfigError, match=r"placeholder for: apiKey"):
FirebaseConfig.load(_write(tmp_path / "firebase.json", unfilled))
def test_reads_a_password(tmp_path: Path) -> None:
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
password = tmp_path / "password"
password.write_text("hunter2", encoding="utf-8")
assert config.read_password(password) == "hunter2"
def test_strips_a_trailing_newline_from_the_password(tmp_path: Path) -> None:
"""The likeliest hand-editing mistake, and otherwise invisible."""
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
password = tmp_path / "password"
password.write_text("hunter2\n", encoding="utf-8")
assert config.read_password(password) == "hunter2"
def test_rejects_an_empty_password(tmp_path: Path) -> None:
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
password = tmp_path / "password"
password.write_text(" ", encoding="utf-8")
with pytest.raises(ConfigError, match="empty or still holds"):
config.read_password(password)
def test_rejects_a_placeholder_password(tmp_path: Path) -> None:
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
password = tmp_path / "password"
password.write_text("PASTE_SYNC_ACCOUNT_PASSWORD_HERE", encoding="utf-8")
with pytest.raises(ConfigError, match="placeholder"):
config.read_password(password)
def test_reports_an_unreadable_password(tmp_path: Path) -> None:
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
(tmp_path / "password").mkdir()
with pytest.raises(ConfigError, match="could not be read"):
config.read_password(tmp_path / "password")
def test_credential_store_is_per_app() -> None:
"""Two apps must not share one cache file; concurrent writes corrupt it."""
diet = credential_store_for("diet_guard")
alarm = credential_store_for("wake_alarm")
assert diet._path != alarm._path
assert diet._path.name == "firebase_auth.json"
assert "diet_guard" in str(diet._path)
def test_client_signs_in_only_without_a_cached_session(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The common path is a cached refresh token, costing no round trip."""
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
sign_ins: list[tuple[str, str]] = []
class _Auth:
def __init__(self, api_key: str, store: object) -> None:
self.api_key = api_key
def has_session(self) -> bool:
return True
def sign_in(self, email: str, password: str) -> None:
sign_ins.append((email, password)) # pragma: no cover - must not run
monkeypatch.setattr("crdt_sync._config.FirebaseTokenProvider", _Auth)
client = firebase_client_for("diet_guard", config=config)
assert sign_ins == []
assert client is not None
def test_client_signs_in_when_there_is_no_session(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
password = tmp_path / "password"
password.write_text("hunter2", encoding="utf-8")
monkeypatch.setattr("crdt_sync._config.PASSWORD_FILE", password)
sign_ins: list[tuple[str, str]] = []
class _Auth:
def __init__(self, api_key: str, store: object) -> None:
self.api_key = api_key
def has_session(self) -> bool:
return False
def sign_in(self, email: str, password_value: str) -> None:
sign_ins.append((email, password_value))
monkeypatch.setattr("crdt_sync._config.FirebaseTokenProvider", _Auth)
firebase_client_for("diet_guard", config=config)
assert sign_ins == [("sync@example.com", "hunter2")]
def test_mirror_client_makes_firebase_primary(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Firebase authoritative, GitHub best-effort -- the cutover shape."""
config = FirebaseConfig.load(_write(tmp_path / "firebase.json", _VALID))
github = _StubRemote()
class _Auth:
def __init__(self, api_key: str, store: object) -> None:
self.api_key = api_key
def has_session(self) -> bool:
return True
def sign_in(self, email: str, password: str) -> None:
raise AssertionError # pragma: no cover - must not run
monkeypatch.setattr("crdt_sync._config.FirebaseTokenProvider", _Auth)
client = mirror_client_for("diet_guard", github, config=config)
# Reaching into the wrapper is the point: which backend is authoritative
# is the one thing a rollback depends on getting right.
assert client.mirror is github
assert isinstance(client.primary, FirebaseSyncClient)
def test_client_loads_the_shared_config_when_none_is_given(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The default path: apps name themselves and nothing else."""
monkeypatch.setattr(
"crdt_sync._config.CONFIG_FILE",
_write(tmp_path / "firebase.json", _VALID),
)
class _Auth:
def __init__(self, api_key: str, store: object) -> None:
self.api_key = api_key
def has_session(self) -> bool:
return True
def sign_in(self, email: str, password: str) -> None:
raise AssertionError # pragma: no cover - must not run
monkeypatch.setattr("crdt_sync._config.FirebaseTokenProvider", _Auth)
assert firebase_client_for("wake_alarm") is not None

View File

@ -1,6 +1,6 @@
[project]
name = "crdt-sync"
version = "0.4.0"
version = "0.5.0"
description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps"
requires-python = ">=3.10"
dependencies = ["requests"]

View File

@ -0,0 +1,80 @@
"""Write the Python half of the live cross-language interop check.
``crdt_sync_dart/tool/interop_check.dart`` merges a Dart record into
``_interop/devices`` and then asserts it can also see a record written by the
*Python* client -- but nothing wrote that record, so the check could only ever
fail. This is the missing half: it pushes ``py-rec`` through the real
:class:`FirebaseSyncClient`, so the Dart tool has something to find.
The two implementations must agree byte-for-byte, or a phone running Dart
would silently not see a PC running Python. That is the failure this pair of
tools exists to catch before any real data depends on it.
Run this first, then the Dart checker::
python3 tool/interop_seed.py
dart run tool/interop_check.dart
Writes only under ``_interop/``; the Dart side deletes both devices' data on
its way out.
"""
from __future__ import annotations
import logging
import sys
from crdt_sync import (
FirebaseConfig,
Hlc,
MemorySyncStateStore,
Record,
dump_log,
firebase_client_for,
load_log,
sync_log,
)
_PATH_PREFIX = "_interop/devices"
_DEVICE_ID = "pydev"
_logger = logging.getLogger("interop-seed")
# The library's canonical serializers, deliberately -- the point of the check
# is that what the real apps write is what the other language reads, so
# reimplementing the encoding here would test the wrong thing.
def main() -> int:
"""Push the Python-side record, returning a process exit code."""
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
client = firebase_client_for("interop", config=FirebaseConfig.load())
merged = sync_log(
client=client,
device_id=_DEVICE_ID,
path_prefix=_PATH_PREFIX,
local_log={
"py-rec": Record(
id="py-rec",
fields={
"value": (
"written-by-python",
Hlc(wall_time_ms=1000, counter=0, node_id="node-py"),
)
},
)
},
encode=dump_log,
decode=load_log,
state_store=MemorySyncStateStore(),
)
_logger.info("python sees records: %s", sorted(merged))
_logger.info("seeded %s; now run: dart run tool/interop_check.dart", _DEVICE_ID)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,212 @@
"""Validate ``~/.config/crdt-sync`` before anything is migrated.
Every failure this catches is one that would otherwise surface *after* data
had started moving, when it is expensive: a placeholder left unreplaced, a
database URL with a trailing slash, a password saved with the trailing
newline a text editor adds, or -- the worst one -- an account whose uid is not
the uid pinned in the security rules, which denies every read and write with
no other symptom.
Checks are ordered cheapest-first and stop at the first failure, so a typo is
reported without a network round trip.
Usage::
python3 tool/preflight_firebase.py
Prints a PASS/FAIL line per check and exits non-zero if any failed. Reads
only; it writes nothing, locally or remotely.
"""
from __future__ import annotations
import base64
import json
import logging
from pathlib import Path
import sys
import requests
from crdt_sync import FirebaseTokenProvider, MemoryCredentialStore
_CONFIG_DIR = Path.home() / ".config" / "crdt-sync"
_CONFIG_FILE = _CONFIG_DIR / "firebase.json"
_PASSWORD_FILE = _CONFIG_DIR / "password"
_REQUIRED_KEYS = ("apiKey", "databaseUrl", "projectId", "uid", "email")
_TIMEOUT_SECONDS = 30
_logger = logging.getLogger("preflight")
class PreflightError(Exception):
"""A configuration problem that must be fixed before migrating."""
def _load_config() -> dict[str, str]:
"""Return the parsed config, or raise with a fixable message."""
if not _CONFIG_FILE.is_file():
msg = f"{_CONFIG_FILE} does not exist"
raise PreflightError(msg)
try:
data = json.loads(_CONFIG_FILE.read_text(encoding="utf-8"))
except ValueError as exc:
msg = f"{_CONFIG_FILE} is not valid JSON: {exc}"
raise PreflightError(msg) from exc
if not isinstance(data, dict):
msg = f"{_CONFIG_FILE} must contain a JSON object"
raise PreflightError(msg)
# The scaffold ships explanatory "_comment_*" keys; they are documentation
# for whoever fills the file in, not configuration.
return {k: v for k, v in data.items() if not k.startswith("_comment")}
def _load_password() -> str:
"""Return the sync account's password, or raise with a fixable message."""
password = (
_PASSWORD_FILE.read_text(encoding="utf-8") if _PASSWORD_FILE.is_file() else ""
)
if not password:
msg = f"{_PASSWORD_FILE} is missing or empty"
raise PreflightError(msg)
return password
def check_keys_present(config: dict[str, str]) -> None:
"""Every required key exists and is a non-empty string."""
missing = [k for k in _REQUIRED_KEYS if not str(config.get(k, "")).strip()]
if missing:
msg = f"missing or empty in firebase.json: {', '.join(missing)}"
raise PreflightError(msg)
def check_no_placeholders(config: dict[str, str], password: str) -> None:
"""No scaffold placeholder survived into a real value."""
unfilled = [k for k, v in config.items() if "PASTE_" in str(v)]
if "PASTE_" in password:
unfilled.append("password (the password file)")
if unfilled:
msg = f"still holding the scaffold placeholder: {', '.join(unfilled)}"
raise PreflightError(msg)
def check_shapes(config: dict[str, str], password: str) -> None:
"""Values look like what they claim to be.
A trailing slash on ``databaseUrl`` yields a double slash in every request
path, and a trailing newline on the password is the single most likely
hand-editing mistake -- both produce confusing auth or 404 errors much
later.
"""
url = config["databaseUrl"]
if not url.startswith("https://"):
msg = f"databaseUrl must start with https:// -- got {url!r}"
raise PreflightError(msg)
if url.endswith("/"):
msg = f"databaseUrl must not end with '/' -- got {url!r}"
raise PreflightError(msg)
if not config["apiKey"].startswith("AIza"):
msg = "apiKey does not start with 'AIza'; that is the Web API key format"
raise PreflightError(msg)
if password != password.strip():
msg = "the password file has leading/trailing whitespace (likely a newline)"
raise PreflightError(msg)
if "@" not in config["email"]:
msg = f"email does not look like an address -- got {config['email']!r}"
raise PreflightError(msg)
def check_rules_deny_anonymous(config: dict[str, str]) -> None:
"""The URL names a real database, and an anonymous read is refused.
Both failures look alike from a distance and are worth separating. A
wrong-region URL answers 404 with a ``correctUrl`` body rather than an
error a caller would notice, so it would otherwise sail past this check,
pass sign-in (a different host entirely) and only break once the
migration started writing.
"""
response = requests.get(f"{config['databaseUrl']}/.json", timeout=_TIMEOUT_SECONDS)
if response.status_code == requests.codes.not_found:
try:
correct = response.json().get("correctUrl")
except ValueError:
correct = None
if correct:
msg = f"databaseUrl is for the wrong region; use {correct!r}"
raise PreflightError(msg)
msg = (
f"{config['databaseUrl']} returned 404 -- no database lives there. "
"Check the URL against Realtime Database > Data in the console."
)
raise PreflightError(msg)
if response.status_code == requests.codes.ok:
msg = (
"UNAUTHENTICATED READ SUCCEEDED -- the database is world-readable. "
"Publish database.rules.json before migrating anything."
)
raise PreflightError(msg)
def check_sign_in_uid_matches(config: dict[str, str], password: str) -> None:
"""Sign-in works and returns exactly the uid pinned in the rules.
The decisive check. A different uid authenticates fine and then fails
every single read and write with a bare permission error.
"""
auth = FirebaseTokenProvider(config["apiKey"], MemoryCredentialStore())
# Raises FirebaseAuthError on bad credentials, or if Email/Password
# sign-in is disabled for the project.
auth.sign_in(config["email"], password)
# sign_in() returns None and keeps only the tokens, so read the uid back
# out of the ID token itself. The token is a JWT whose payload carries the
# subject; decoding it needs no verification here, because the value is
# only being compared against the local config, not trusted for access.
payload = auth.id_token().split(".")[1]
padded = payload + "=" * (-len(payload) % 4)
claims = json.loads(base64.urlsafe_b64decode(padded))
actual = claims.get("user_id") or claims.get("sub")
if actual != config["uid"]:
msg = (
f"signed in as uid {actual!r} but the rules pin {config['uid']!r} -- "
"every read and write would be denied"
)
raise PreflightError(msg)
def main() -> int:
"""Run every check, returning a process exit code."""
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
try:
config = _load_config()
password = _load_password()
checks = (
("config keys present", lambda: check_keys_present(config)),
("no placeholders left", lambda: check_no_placeholders(config, password)),
("values well-formed", lambda: check_shapes(config, password)),
("rules deny anonymous reads", lambda: check_rules_deny_anonymous(config)),
(
"sign-in uid matches rules",
lambda: check_sign_in_uid_matches(config, password),
),
)
for name, run in checks:
run()
_logger.info(" PASS %s", name)
except PreflightError as exc:
_logger.info(" FAIL %s", exc)
return 1
except (requests.RequestException, OSError) as exc:
_logger.info(" FAIL could not reach Firebase: %s", exc)
return 1
_logger.info("\nAll preflight checks passed; safe to run the migration.")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -3,6 +3,7 @@ library;
export 'src/firebase_auth_rest.dart';
export 'src/firebase_client.dart';
export 'src/firebase_config.dart';
export 'src/github_client.dart';
export 'src/hlc.dart';
export 'src/log.dart';

View File

@ -0,0 +1,155 @@
/// The Dart counterpart to Python's `crdt_sync._config`.
///
/// Both languages talk to one Firebase project as one account, so the shape
/// of the configuration is shared even though where it is *stored* is not:
/// on Linux it is `~/.config/crdt-sync/firebase.json` (mode 0600), while on
/// Android the password and refresh token belong in `flutter_secure_storage`
/// and the file does not exist at all.
///
/// [FirebaseConfig] therefore takes plain values and stays free of `dart:io`,
/// so it works on web and mobile. The platform-specific loading is the app's
/// job -- `crdt_sync_io.dart` provides the desktop half.
library;
import 'dart:convert';
import 'package:http/http.dart' as http;
import 'firebase_auth_rest.dart';
import 'firebase_client.dart';
import 'mirror_store.dart';
import 'remote_store.dart';
/// The shared config is missing a field, or still holds a scaffold placeholder.
class ConfigException implements Exception {
ConfigException(this.message);
final String message;
@override
String toString() => 'ConfigException: $message';
}
/// Everything needed to reach the shared Firebase project.
///
/// [apiKey] is not a secret: it ships inside the Android APKs, and the
/// security rules are what protect the data. The password is.
class FirebaseConfig {
const FirebaseConfig({
required this.apiKey,
required this.databaseUrl,
required this.projectId,
required this.uid,
required this.email,
});
/// Parses the same `firebase.json` the Python side reads.
///
/// Throws [ConfigException] naming the field at fault, because the
/// alternative is an authentication failure much later with no clue which
/// value was wrong. Keys prefixed `_comment` are the scaffold's inline
/// documentation, not configuration, and are ignored.
factory FirebaseConfig.fromJson(Map<String, dynamic> json) {
String require(String key) {
final value = json[key];
if (value == null || (value is String && value.trim().isEmpty)) {
throw ConfigException('firebase.json is missing or has empty: $key');
}
final text = value.toString();
if (text.contains('PASTE_')) {
throw ConfigException(
'firebase.json still holds the placeholder for: $key',
);
}
return text;
}
return FirebaseConfig(
apiKey: require('apiKey'),
databaseUrl: require('databaseUrl'),
projectId: require('projectId'),
uid: require('uid'),
email: require('email'),
);
}
/// Parses [text] as the shared config file's contents.
factory FirebaseConfig.parse(String text) {
final Object? decoded;
try {
decoded = jsonDecode(text);
} on FormatException catch (error) {
throw ConfigException(
'firebase.json is not valid JSON: ${error.message}',
);
}
if (decoded is! Map<String, dynamic>) {
throw ConfigException('firebase.json must contain a JSON object');
}
return FirebaseConfig.fromJson(decoded);
}
final String apiKey;
final String databaseUrl;
final String projectId;
/// The uid the security rules pin. A session for any other uid authenticates
/// fine and is then denied every read and write.
final String uid;
final String email;
}
/// Returns a client for [config], signing in only when there is no session.
///
/// The common path costs no authentication round trip: [store] holds the
/// refresh token between launches. Pass the platform-appropriate store --
/// `flutter_secure_storage` on Android, a 0600 file on desktop.
Future<FirebaseRestClient> firebaseClientFor({
required FirebaseConfig config,
required FirebaseCredentialStore store,
String? password,
http.Client? httpClient,
}) async {
final auth = FirebaseTokenProvider(
apiKey: config.apiKey,
store: store,
httpClient: httpClient,
);
if (await store.load() == null) {
if (password == null) {
throw ConfigException(
'no stored session for ${config.email} and no password given; '
'sign in once from the settings screen',
);
}
await auth.signIn(email: config.email, password: password);
}
return FirebaseRestClient(
databaseUrl: config.databaseUrl,
auth: auth,
httpClient: httpClient,
);
}
/// Returns a Firebase-primary store that still mirrors to [githubClient].
///
/// What an app uses *during* the cutover: Firebase is authoritative, GitHub is
/// kept in step but never allowed to fail a tick, and reads union both so
/// devices can cut over one at a time. Rolling back is passing [githubClient]
/// straight to `syncLog` again; retiring the mirror is calling
/// [firebaseClientFor] instead. Both are one-line changes at the call site.
Future<MirrorStore> mirrorStoreFor({
required FirebaseConfig config,
required FirebaseCredentialStore store,
required RemoteStore githubClient,
String? password,
http.Client? httpClient,
}) async => MirrorStore(
primary: await firebaseClientFor(
config: config,
store: store,
password: password,
httpClient: httpClient,
),
mirror: githubClient,
);

View File

@ -5,6 +5,7 @@ import 'package:crypto/crypto.dart';
import 'log.dart';
import 'record.dart';
import 'remote_store.dart';
import 'store.dart';
const _defaultFilename = 'log.json';
@ -66,6 +67,57 @@ class InMemorySyncStateStore implements SyncStateStore {
Future<void> save(SyncState state) async => _state = state;
}
/// A [SyncStateStore] that persists through a [LogPersistence] port.
///
/// This is the mobile counterpart to Python's `FileSyncStateStore`, and it is
/// what makes the revision saving real on a phone: an app is a fresh process
/// after every cold start, so an [InMemorySyncStateStore] would forget every
/// peer revision and re-download all of them on each launch -- the exact
/// traffic the revision scheme exists to avoid.
///
/// Takes the pure-Dart [LogPersistence] port rather than a `File` so it works
/// on web (where `dart:io` is unavailable) and is trivially faked in tests.
/// On mobile and desktop, pass a `FileLogPersistence` from
/// `package:crdt_sync/crdt_sync_io.dart`, which already writes atomically.
///
/// **Store this next to the log it describes and clear the two together.**
/// Skipping an unchanged peer is only sound because that peer's records are
/// already merged into the local log; state that outlived its log would skip
/// peers whose data had been lost. See [SyncState].
class PersistedSyncStateStore implements SyncStateStore {
/// Persists state through [persistence].
PersistedSyncStateStore(this._persistence);
final LogPersistence _persistence;
/// Returns the stored state, or a default one if absent or unreadable.
///
/// A corrupt or half-written file degrades to "remember nothing", which
/// costs one tick of extra traffic rather than failing the sync -- the same
/// fail-safe choice the Python side makes.
@override
Future<SyncState> load() async {
final String? text;
try {
text = await _persistence.read();
} on Exception {
return const SyncState();
}
if (text == null || text.isEmpty) return const SyncState();
try {
final decoded = jsonDecode(text);
if (decoded is! Map<String, dynamic>) return const SyncState();
return SyncState.fromJson(decoded);
} on FormatException {
return const SyncState();
}
}
@override
Future<void> save(SyncState state) async =>
_persistence.write(jsonEncode(state.toJson()));
}
/// The revision of a serialized log: a content hash, not a clock reading.
///
/// A hash rather than the log's maximum HLC because merging a peer's *older*
@ -161,11 +213,7 @@ Future<Log> syncLog({
if (stateStore != null) {
// Published after the log, never before: a peer that cached "seen rev
// X" against a log it never received would skip it forever.
await client.putFileText(
'$revs/$deviceId',
rev,
message: commitMessage,
);
await client.putFileText('$revs/$deviceId', rev, message: commitMessage);
}
}
await stateStore?.save(SyncState(pushedRev: rev, peerRevs: seenRevs));

View File

@ -2,7 +2,7 @@ name: crdt_sync
description: >-
Shared CRDT merge scheme + GitHub-Contents-API sync transport for
cross-device apps. Pure Dart, no Flutter dependency.
version: 0.4.0
version: 0.5.0
publish_to: 'none'
environment:

View File

@ -0,0 +1,232 @@
/// Tests for the shared Firebase configuration on the Dart side.
///
/// The failure modes asserted here are the ones that would otherwise surface
/// as an authentication error long after the real mistake, so each is checked
/// to name the field at fault rather than merely to throw.
library;
import 'dart:convert';
import 'package:crdt_sync/crdt_sync.dart';
import 'package:http/http.dart' as http;
import 'package:http/testing.dart' as http_testing;
import 'package:test/test.dart';
const _valid = {
'apiKey': 'AIzaSyExample',
'databaseUrl':
'https://kuhy-syncs-default-rtdb.europe-west1.firebasedatabase.app',
'projectId': 'kuhy-syncs',
'uid': 'OvA2REQyLIhAHOEjzwS1o877rgG3',
'email': 'sync@example.com',
};
/// A [RemoteStore] that does nothing, standing in for the GitHub client.
class _StubRemote implements RemoteStore {
@override
Future<List<String>> listDirectory(String path) async => [];
@override
Future<String?> getFileText(String path) async => null;
@override
Future<void> putFileText(
String path,
String text, {
required String message,
}) async {}
@override
Future<void> deleteFile(String path, {String message = ''}) async {}
@override
Future<bool> canAccessRemote() async => true;
@override
void close() {}
}
void main() {
group('FirebaseConfig', () {
test('parses a valid config', () {
final config = FirebaseConfig.parse(jsonEncode(_valid));
expect(config.apiKey, 'AIzaSyExample');
expect(config.projectId, 'kuhy-syncs');
expect(config.uid, 'OvA2REQyLIhAHOEjzwS1o877rgG3');
expect(config.email, 'sync@example.com');
expect(config.databaseUrl, endsWith('europe-west1.firebasedatabase.app'));
});
test('ignores the scaffold comment keys', () {
final annotated = {..._valid, '_comment_apiKey': 'where to find it'};
expect(
FirebaseConfig.parse(jsonEncode(annotated)).apiKey,
'AIzaSyExample',
);
});
test('agrees with the Python side on the field names', () {
// Both languages read the same file; a rename on one side only would
// break the other silently.
final config = FirebaseConfig.parse(jsonEncode(_valid));
expect(config.databaseUrl, _valid['databaseUrl']);
expect(config.uid, _valid['uid']);
});
for (final field in _valid.keys) {
test('names $field when it is missing', () {
final incomplete = {..._valid}..remove(field);
expect(
() => FirebaseConfig.parse(jsonEncode(incomplete)),
throwsA(
isA<ConfigException>().having(
(e) => e.message,
'message',
contains(field),
),
),
);
});
}
test('names an empty field', () {
final blank = {..._valid, 'uid': ' '};
expect(
() => FirebaseConfig.parse(jsonEncode(blank)),
throwsA(
isA<ConfigException>().having(
(e) => e.message,
'message',
contains('uid'),
),
),
);
});
test('names an unfilled placeholder', () {
final unfilled = {..._valid, 'apiKey': 'PASTE_WEB_API_KEY_HERE'};
expect(
() => FirebaseConfig.parse(jsonEncode(unfilled)),
throwsA(
isA<ConfigException>().having(
(e) => e.message,
'message',
contains('placeholder for: apiKey'),
),
),
);
});
test('rejects unparsable JSON', () {
expect(
() => FirebaseConfig.parse('{not json'),
throwsA(isA<ConfigException>()),
);
});
test('rejects a non-object document', () {
expect(
() => FirebaseConfig.parse('["a list"]'),
throwsA(isA<ConfigException>()),
);
});
test('has a readable toString on the exception', () {
expect(ConfigException('boom').toString(), contains('boom'));
});
});
group('client construction', () {
test('refuses to build without a session or a password', () async {
// Better than signing in with an empty password and failing remotely.
await expectLater(
firebaseClientFor(
config: FirebaseConfig.parse(jsonEncode(_valid)),
store: InMemoryCredentialStore(),
),
throwsA(isA<ConfigException>()),
);
});
test('signs in with the password when there is no session', () async {
// The first-run path: no cached refresh token, so it must exchange the
// password for one rather than failing.
final store = InMemoryCredentialStore();
final sentTo = <String>[];
final mock = http_testing.MockClient((request) async {
sentTo.add(request.url.path);
return http.Response(
jsonEncode({
'idToken': 'id',
'refreshToken': 'refresh',
'expiresIn': '3600',
'localId': _valid['uid'],
}),
200,
);
});
final client = await firebaseClientFor(
config: FirebaseConfig.parse(jsonEncode(_valid)),
store: store,
password: 'hunter2',
httpClient: mock,
);
expect(sentTo.single, contains('signInWithPassword'));
expect(await store.load(), isNotNull, reason: 'session must persist');
client.close();
});
test('reuses a stored session without a password', () async {
final store = InMemoryCredentialStore();
await store.save(
FirebaseCredentials(
idToken: 'id',
refreshToken: 'refresh',
expiresAt: DateTime.now().add(const Duration(hours: 1)),
),
);
final client = await firebaseClientFor(
config: FirebaseConfig.parse(jsonEncode(_valid)),
store: store,
);
expect(client, isA<FirebaseRestClient>());
client.close();
});
test(
'mirrorStoreFor makes Firebase primary and GitHub the mirror',
() async {
// Which backend is authoritative is the one thing a rollback depends on.
final store = InMemoryCredentialStore();
await store.save(
FirebaseCredentials(
idToken: 'id',
refreshToken: 'refresh',
expiresAt: DateTime.now().add(const Duration(hours: 1)),
),
);
final github = _StubRemote();
final mirror = await mirrorStoreFor(
config: FirebaseConfig.parse(jsonEncode(_valid)),
store: store,
githubClient: github,
);
expect(mirror.primary, isA<FirebaseRestClient>());
expect(mirror.mirror, same(github));
mirror.close();
},
);
});
}

View File

@ -99,8 +99,7 @@ class _FakeRemoteWithoutBulkRead implements RemoteStore {
List<String> get writes => _inner.writes;
@override
Future<List<String>> listDirectory(String path) =>
_inner.listDirectory(path);
Future<List<String>> listDirectory(String path) => _inner.listDirectory(path);
@override
Future<String?> getFileText(String path) => _inner.getFileText(path);
@ -151,7 +150,10 @@ void main() {
group('defaultRevsPath', () {
test('is a sibling of the device directory', () {
expect(defaultRevsPath('diet-guard-sync/devices'), 'diet-guard-sync/revs');
expect(
defaultRevsPath('diet-guard-sync/devices'),
'diet-guard-sync/revs',
);
expect(defaultRevsPath('todo-sync/notes'), 'todo-sync/revs');
});
@ -184,10 +186,7 @@ void main() {
final remote = _FakeRemote({});
final store = InMemorySyncStateStore();
await _tick(remote, store, local: _log('a', '1'));
expect(remote.writes, [
'ns/devices/pc/log.json',
'ns/revs/pc',
]);
expect(remote.writes, ['ns/devices/pc/log.json', 'ns/revs/pc']);
});
test('a second unchanged tick writes nothing at all', () async {
@ -217,10 +216,10 @@ void main() {
final local = _log('a', '1');
await _tick(remote, null, local: local);
await _tick(remote, null, local: local);
expect(
remote.writes,
['ns/devices/pc/log.json', 'ns/devices/pc/log.json'],
);
expect(remote.writes, [
'ns/devices/pc/log.json',
'ns/devices/pc/log.json',
]);
// No state store means no revision publishing either.
expect(remote.writes.where((w) => w.startsWith('ns/revs')), isEmpty);
});
@ -357,4 +356,96 @@ void main() {
expect(remote.files['ns/revs/phone'], 'peer-rev');
});
});
group('PersistedSyncStateStore', () {
test('round-trips state through persistence', () async {
final persistence = _FakePersistence();
const state = SyncState(
pushedRev: 'rev-local',
peerRevs: {'phone': 'rev-phone'},
);
await PersistedSyncStateStore(persistence).save(state);
final loaded = await PersistedSyncStateStore(persistence).load();
// A *separate* store instance reads it back: that is the whole point --
// surviving the cold start that an in-memory store does not.
expect(loaded.pushedRev, 'rev-local');
expect(loaded.peerRevs, {'phone': 'rev-phone'});
});
test('survives a cold start, so peers are not re-downloaded', () async {
// The free-tier claim on mobile rests on this: a fresh process must
// still skip an unchanged peer.
final persistence = _FakePersistence();
final remote = _FakeRemote({
'ns/devices/phone/log.json': _encode(_log('a', '1')),
'ns/revs/phone': revisionOf(_encode(_log('a', '1'))),
});
await _tick(remote, PersistedSyncStateStore(persistence));
remote.reads.clear();
await _tick(remote, PersistedSyncStateStore(persistence));
expect(remote.reads, isNot(contains('ns/devices/phone/log.json')));
});
test('reports nothing remembered when absent', () async {
final loaded = await PersistedSyncStateStore(_FakePersistence()).load();
expect(loaded.pushedRev, isNull);
expect(loaded.peerRevs, isEmpty);
});
test('degrades to remembering nothing when empty', () async {
final loaded = await PersistedSyncStateStore(
_FakePersistence(text: ''),
).load();
expect(loaded.pushedRev, isNull);
});
test('degrades to remembering nothing when corrupt', () async {
// Costs one tick of extra traffic rather than failing the sync.
final loaded = await PersistedSyncStateStore(
_FakePersistence(text: 'not json{'),
).load();
expect(loaded.pushedRev, isNull);
});
test('degrades to remembering nothing when not an object', () async {
final loaded = await PersistedSyncStateStore(
_FakePersistence(text: '["a list"]'),
).load();
expect(loaded.pushedRev, isNull);
});
test('degrades to remembering nothing when the read throws', () async {
// An unreadable file must not fail the tick.
final loaded = await PersistedSyncStateStore(
_FakePersistence(readThrows: true),
).load();
expect(loaded.pushedRev, isNull);
});
});
}
/// A [LogPersistence] fake with injectable failure modes.
class _FakePersistence implements LogPersistence {
_FakePersistence({this.text, this.readThrows = false});
String? text;
final bool readThrows;
@override
Future<String?> read() async {
if (readThrows) throw const FormatException('unreadable');
return text;
}
@override
Future<void> write(String value) async => text = value;
}