From cfc4c295e44058023151fade735a9b2f76090798 Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Mon, 3 Aug 2026 18:31:18 +0200 Subject: [PATCH] crdt_sync: add the one-shot GitHub -> Firebase data migration Reads through the GitHub API, not a local clone: the checkout on this machine was five days stale while devices kept syncing, so seeding from it would have silently rolled back everything written in between. Blobs are fetched by sha rather than by path, so a concurrent push cannot swap the content out from under the tree listing. Writes through the real FirebaseSyncClient so the migration exercises the same key escaping the apps will, then reads every blob back and compares bytes before reporting success -- a migration that claimed success without checking would be indistinguishable from one that dropped a file. Retries connection resets and 5xx: GitHub drops keep-alive connections routinely, and a half-completed migration is worse than one that never started. Three things are deliberately left behind, and stay in the GitHub repo (archived, not deleted): - todo-sync/changesets/ pre-migration sqlite_crdt format, unread since 2026-07-17 - diet-guard-sync/devices/desktop/ a replica that stopped syncing on 2026-07-22; verified to hold 0 records, 0 tombstones and 0 fields newer than both `pc` and `phone`. Carrying it over would cost a full re-download every tick forever, since a device that publishes no revision can never be skipped - READMEs and .gitkeep, which are not sync data Run against the live database: 16 files, 956,272 bytes, all verified byte-for-byte, and independently re-checked by listing the namespaces and parsing records back out. --- crdt-sync/tool/__init__.py | 6 + crdt-sync/tool/migrate_github_to_firebase.py | 216 +++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 crdt-sync/tool/__init__.py create mode 100644 crdt-sync/tool/migrate_github_to_firebase.py diff --git a/crdt-sync/tool/__init__.py b/crdt-sync/tool/__init__.py new file mode 100644 index 0000000..971c6eb --- /dev/null +++ b/crdt-sync/tool/__init__.py @@ -0,0 +1,6 @@ +"""One-shot operational scripts for crdt_sync. + +A package rather than a loose directory so the linter treats these as real +modules; they are not imported by the library and are excluded from its +coverage measurement. +""" diff --git a/crdt-sync/tool/migrate_github_to_firebase.py b/crdt-sync/tool/migrate_github_to_firebase.py new file mode 100644 index 0000000..94b4c14 --- /dev/null +++ b/crdt-sync/tool/migrate_github_to_firebase.py @@ -0,0 +1,216 @@ +"""One-shot migration of the GitHub-backed sync repo into Firebase RTDB. + +Reads through the GitHub API rather than a local clone: a checkout goes stale +the moment any device syncs, and seeding Firebase from stale data would +silently roll back whatever changed in between. + +Writes through the real :class:`crdt_sync.FirebaseSyncClient`, so the +migration exercises the same key escaping and error handling the apps will, +then reads every blob back and compares bytes before reporting success. + +Usage:: + + python3 tool/migrate_github_to_firebase.py --dry-run + python3 tool/migrate_github_to_firebase.py + +Configuration comes from ``~/.config/crdt-sync/firebase.json`` and the GitHub +token from ``~/.config/diet_guard/sync_token``. +""" + +from __future__ import annotations + +import argparse +import base64 +from dataclasses import dataclass +import json +import logging +from pathlib import Path +import sys + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from crdt_sync import ( + FirebaseSyncClient, + FirebaseTokenProvider, + MemoryCredentialStore, +) + +_CONFIG_DIR = Path.home() / ".config" / "crdt-sync" +_GITHUB_TOKEN_FILE = Path.home() / ".config" / "diet_guard" / "sync_token" +_REPO = "kuhyx/syncs" +_API = "https://api.github.com" +_TIMEOUT_SECONDS = 30 + +# A migration that reports nothing is indistinguishable from one that did +# nothing, so progress goes to stdout -- via logging rather than print, which +# this project's lint config reserves for accidental debug output. +_logger = logging.getLogger("migrate") + +# Paths deliberately left behind, each for a reason that would otherwise cost +# egress or storage forever. They stay in the GitHub repo, which is archived +# rather than deleted, so nothing is destroyed by skipping them. +_SKIP_PREFIXES = ( + # Pre-migration sqlite_crdt format, last written 2026-07-17. Nothing + # reads it; `sync_service.dart` only touches `notes/`. + "todo-sync/changesets/", + # A replica that stopped syncing 2026-07-22: verified to hold 0 records, + # 0 tombstones and 0 newer fields that `pc` and `phone` lack. Carrying it + # over would cost a full re-download every tick forever, because a device + # that publishes no revision can never be skipped. + "diet-guard-sync/devices/desktop/", +) + +# Not sync data -- documentation and directory placeholders. +_SKIP_NAMES = ("README.md", ".gitkeep") + + +@dataclass(frozen=True) +class Blob: + """One file in the sync repo, with its exact bytes.""" + + path: str + text: str + + @property + def size(self) -> int: + """Return the payload size in bytes.""" + return len(self.text.encode("utf-8")) + + +def _github_session(token: str) -> requests.Session: + """Return a session that retries transient failures. + + GitHub drops keep-alive connections routinely, and a migration that gave + up half-way would leave Firebase holding a partial copy -- the one + outcome worse than not starting. Retries cover connection resets and 5xx, + plus 403/429 for secondary rate limiting. + """ + session = requests.Session() + session.headers.update( + { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + ) + retry = Retry( + total=5, + backoff_factor=1.0, + status_forcelist=(403, 429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET"}), + raise_on_status=False, + ) + session.mount("https://", HTTPAdapter(max_retries=retry)) + return session + + +def _list_blobs(session: requests.Session) -> list[tuple[str, str]]: + """Return ``(path, sha)`` for every file, via one recursive tree call.""" + response = session.get( + f"{_API}/repos/{_REPO}/git/trees/HEAD", + params={"recursive": "1"}, + timeout=_TIMEOUT_SECONDS, + ) + response.raise_for_status() + tree = response.json() + if tree.get("truncated"): + # Would silently migrate only part of the repo. + msg = "GitHub truncated the tree listing; the repo is too large" + raise RuntimeError(msg) + return [ + (item["path"], item["sha"]) for item in tree["tree"] if item["type"] == "blob" + ] + + +def _fetch_blob(session: requests.Session, path: str, sha: str) -> Blob: + """Return the exact bytes of one blob. + + Uses the blobs API rather than the contents API: it is addressed by sha, + so it cannot race a concurrent push into returning a different file than + the tree listing described. + """ + response = session.get( + f"{_API}/repos/{_REPO}/git/blobs/{sha}", timeout=_TIMEOUT_SECONDS + ) + response.raise_for_status() + payload = response.json() + raw = base64.b64decode(payload["content"]) + return Blob(path=path, text=raw.decode("utf-8")) + + +def _should_migrate(path: str) -> bool: + if any(path.startswith(prefix) for prefix in _SKIP_PREFIXES): + return False + return Path(path).name not in _SKIP_NAMES + + +def _firebase_client() -> FirebaseSyncClient: + config = json.loads((_CONFIG_DIR / "firebase.json").read_text()) + password = (_CONFIG_DIR / "password").read_text() + auth = FirebaseTokenProvider(config["apiKey"], MemoryCredentialStore()) + auth.sign_in(config["email"], password) + return FirebaseSyncClient(config["databaseUrl"], auth) + + +def _report(blobs: list[Blob], skipped: list[str]) -> None: + """Log exactly what will move and what will not, before anything moves.""" + total = sum(blob.size for blob in blobs) + _logger.info("%d files to migrate, %d bytes total", len(blobs), total) + for blob in sorted(blobs, key=lambda b: -b.size): + _logger.info(" %8d %s", blob.size, blob.path) + if skipped: + _logger.info("%d skipped (see _SKIP_PREFIXES for why):", len(skipped)) + for path in skipped: + _logger.info(" %s", path) + + +def main() -> int: + """Run the migration, returning a process exit code.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="list what would be migrated, write nothing", + ) + args = parser.parse_args() + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") + + session = _github_session(_GITHUB_TOKEN_FILE.read_text().strip()) + entries = _list_blobs(session) + blobs = [ + _fetch_blob(session, path, sha) + for path, sha in entries + if _should_migrate(path) + ] + skipped = [path for path, _ in entries if not _should_migrate(path)] + _report(blobs, skipped) + + if args.dry_run: + _logger.info("DRY RUN -- nothing written") + return 0 + + client = _firebase_client() + _logger.info("writing...") + for blob in blobs: + client.put_file_text(blob.path, blob.text, message="migration") + + # Verify by reading back through the same client: a migration that + # reported success without checking would be indistinguishable from one + # that silently dropped a file. + _logger.info("verifying byte-for-byte...") + mismatches = [ + blob.path for blob in blobs if client.get_file_text(blob.path) != blob.text + ] + if mismatches: + _logger.error("FAILED -- %d file(s) did not round-trip:", len(mismatches)) + for path in mismatches: + _logger.error(" %s", path) + return 1 + + _logger.info("OK -- %d files migrated and verified byte-for-byte", len(blobs)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())