From 317448062419e3faf48aee8a50452c5572d244f9 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Thu, 12 Nov 2026 10:44:44 +0100 Subject: [PATCH] crdt-sync 0.11.1: verified plaintext cleanup clear_plaintext_prefix + tool/e2e_cleanup: dry run by default, never creates a header, deletes the old plaintext prefix only when every old record is in the encrypted copy. Todo: fda71ed1 Co-Authored-By: Claude Opus 5.5 --- crdt-sync/crdt_sync/__init__.py | 3 + crdt-sync/crdt_sync/_e2e_cleanup.py | 79 +++++++++++++ crdt-sync/crdt_sync/tests/test_e2e_cleanup.py | 108 ++++++++++++++++++ crdt-sync/pyproject.toml | 2 +- crdt-sync/tool/e2e_cleanup.py | 58 ++++++++++ 5 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 crdt-sync/crdt_sync/_e2e_cleanup.py create mode 100644 crdt-sync/crdt_sync/tests/test_e2e_cleanup.py create mode 100644 crdt-sync/tool/e2e_cleanup.py diff --git a/crdt-sync/crdt_sync/__init__.py b/crdt-sync/crdt_sync/__init__.py index d614d2d..ff55659 100644 --- a/crdt-sync/crdt_sync/__init__.py +++ b/crdt-sync/crdt_sync/__init__.py @@ -21,6 +21,7 @@ from crdt_sync._e2e_cipher import ( open_log_text, seal_log_text, ) +from crdt_sync._e2e_cleanup import clear_plaintext_prefix, unverified_blobs from crdt_sync._e2e_key import ( Argon2Params, E2eHeaderError, @@ -113,6 +114,7 @@ __all__ = [ "SyncState", "SyncStateStore", "SyncTarget", + "clear_plaintext_prefix", "copy_to_encrypted", "credential_store_for", "default_e2e_path", @@ -134,5 +136,6 @@ __all__ = [ "seal_log_text", "sync_log", "unlock_e2e", + "unverified_blobs", "write_log", ] diff --git a/crdt-sync/crdt_sync/_e2e_cleanup.py b/crdt-sync/crdt_sync/_e2e_cleanup.py new file mode 100644 index 0000000..0693841 --- /dev/null +++ b/crdt-sync/crdt_sync/_e2e_cleanup.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Clear an app's old plaintext prefix once its encrypted copy is proven. + +The migration never deletes the old prefix by itself: an old build may still +write there. Run this once every device runs an encrypting build. It never +creates a header (a typo'd passphrase must not lock the app for good) and +deletes nothing unless every old blob is contained in the encrypted copy. +""" + +from __future__ import annotations + +import hmac +from typing import TYPE_CHECKING + +from crdt_sync._e2e_key import derive_key, read_header +from crdt_sync._e2e_store import E2eLayout, E2eLockedError, EncryptingStore, _blobs +from crdt_sync._log import merge_logs +from crdt_sync._store import dump_log, load_log + +if TYPE_CHECKING: + from crdt_sync._e2e_key import E2eKey + from crdt_sync._remote import RemoteStore + + +def unverified_blobs( + store: RemoteStore, key: E2eKey, old_prefix: str, layout: E2eLayout +) -> list[str]: + """Return old blobs (relative paths) not fully contained in the copy.""" + dest = EncryptingStore(store, key, layout) + missing: list[str] = [] + for rel, text in sorted(_blobs(store, old_prefix).items()): + try: + source = load_log(text) + copied = dest.get_file_text(f"{layout.prefix}/{rel}") + current = load_log(copied) if copied is not None else None + except ValueError, KeyError, TypeError: + missing.append(rel) + continue + if not source: + continue + if current is None or dump_log(merge_logs(current, source)) != dump_log( + current + ): + missing.append(rel) + return missing + + +def _existing_key(store: RemoteStore, layout: E2eLayout, passphrase: str) -> E2eKey: + header = read_header(store, layout.e2e_path) + if header is None: + msg = f"no e2e header at {layout.e2e_path}: nothing was migrated" + raise E2eLockedError(msg) + key = derive_key(passphrase, header.salt, header.params) + if not hmac.compare_digest(key.check_for(header.salt), header.check): + msg = "wrong crdt-sync passphrase" + raise E2eLockedError(msg) + return key + + +def clear_plaintext_prefix( + store: RemoteStore, old_prefix: str, *, passphrase: str, apply: bool +) -> list[str]: + """Return the old paths to delete; delete them only when ``apply``. + + Raises: + E2eLockedError: No header, a wrong passphrase, or a copy that does + not yet hold every old record. + """ + layout = E2eLayout.of(old_prefix) + key = _existing_key(store, layout, passphrase) + missing = unverified_blobs(store, key, old_prefix, layout) + if missing: + msg = f"copy not verified for {', '.join(missing)}" + raise E2eLockedError(msg) + paths = [f"{old_prefix}/{rel}" for rel in sorted(_blobs(store, old_prefix))] + if apply: + for path in paths: + store.delete_file(path, message="e2e: clear plaintext prefix") + return paths diff --git a/crdt-sync/crdt_sync/tests/test_e2e_cleanup.py b/crdt-sync/crdt_sync/tests/test_e2e_cleanup.py new file mode 100644 index 0000000..deb0f84 --- /dev/null +++ b/crdt-sync/crdt_sync/tests/test_e2e_cleanup.py @@ -0,0 +1,108 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Clearing an old plaintext prefix only after the encrypted copy is proven.""" + +from __future__ import annotations + +import io +import json + +import pytest + +from crdt_sync import ( + Argon2Params, + E2eLayout, + E2eLockedError, + clear_plaintext_prefix, + copy_to_encrypted, + unlock_e2e, + unverified_blobs, +) +from crdt_sync.tests.conftest import FakeStore +from tool import e2e_cleanup + +CHEAP = Argon2Params(8, 1, 1) +PHRASE = "pw" +WRONG = PHRASE + "x" +OLD = "app/devices" +LAYOUT = E2eLayout.of(OLD) + + +def _log(title: str, stamp: str = "2026-01-01T00:00:00.000Z") -> str: + return json.dumps( + { + "n": { + "id": "n", + "fields": {"t": [title, f"{stamp}-0000-a"]}, + "deleted": False, + "deleted_hlc": None, + } + } + ) + + +def _seeded() -> FakeStore: + remote = FakeStore({f"{OLD}/pc/log.json": _log("x"), "app/revs/pc": "r"}) + key = unlock_e2e( + remote, + path_prefix=LAYOUT.prefix, + passphrase=PHRASE, + params=CHEAP, + e2e_path=LAYOUT.e2e_path, + ) + copy_to_encrypted(remote, key, OLD, LAYOUT) + return remote + + +def test_dry_run_lists_and_deletes_nothing() -> None: + remote = _seeded() + paths = clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False) + assert paths == [f"{OLD}/pc/log.json"] + assert f"{OLD}/pc/log.json" in remote.files + + +def test_apply_deletes_only_after_verification() -> None: + remote = _seeded() + clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=True) + assert f"{OLD}/pc/log.json" not in remote.files + assert f"{LAYOUT.prefix}/pc/log.json" in remote.files + + +def test_unverified_copy_refuses() -> None: + remote = _seeded() + remote.files[f"{OLD}/pc/log.json"] = _log("newer", "2026-02-01T00:00:00.000Z") + remote.files[f"{OLD}/bad/log.json"] = "{oops" + key = unlock_e2e( + remote, path_prefix=LAYOUT.prefix, passphrase=PHRASE, e2e_path=LAYOUT.e2e_path + ) + assert unverified_blobs(remote, key, OLD, LAYOUT) == ["bad/log.json", "pc/log.json"] + with pytest.raises(E2eLockedError, match="not verified"): + clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=True) + assert f"{OLD}/pc/log.json" in remote.files + + +def test_empty_logs_need_no_copy() -> None: + remote = _seeded() + remote.files[f"{OLD}/tab/log.json"] = "{}" + paths = clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False) + assert f"{OLD}/tab/log.json" in paths + + +def test_never_creates_a_header_and_wrong_passphrase_locks() -> None: + remote = FakeStore({f"{OLD}/pc/log.json": _log("x")}) + with pytest.raises(E2eLockedError, match="no e2e header"): + clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False) + assert LAYOUT.e2e_path not in remote.files + seeded = _seeded() + with pytest.raises(E2eLockedError): + clear_plaintext_prefix(seeded, OLD, passphrase=WRONG, apply=False) + + +def test_tool_main(monkeypatch: pytest.MonkeyPatch) -> None: + remote = _seeded() + out = io.StringIO() + monkeypatch.setattr("sys.stdout", out) + assert e2e_cleanup.run(remote, OLD, PHRASE, apply=False) == 0 + assert "would delete 1" in out.getvalue() + assert e2e_cleanup.run(remote, OLD, PHRASE, apply=True) == 0 + assert "deleted 1" in out.getvalue() + assert e2e_cleanup.run(FakeStore(), OLD, PHRASE, apply=True) == 1 diff --git a/crdt-sync/pyproject.toml b/crdt-sync/pyproject.toml index e71c14d..8ce17bb 100644 --- a/crdt-sync/pyproject.toml +++ b/crdt-sync/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "crdt-sync" -version = "0.11.0" +version = "0.11.1" description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps" requires-python = ">=3.14" dependencies = [ diff --git a/crdt-sync/tool/e2e_cleanup.py b/crdt-sync/tool/e2e_cleanup.py new file mode 100644 index 0000000..01adde3 --- /dev/null +++ b/crdt-sync/tool/e2e_cleanup.py @@ -0,0 +1,58 @@ +# Copyright (c) 2026 Krzysztof Rudnicki +"""Clear an app's old plaintext RTDB prefix after the E2E copy is verified. + +Dry run by default; ``--apply`` deletes. Run only once every device runs an +encrypting build:: + + python3 -m tool.e2e_cleanup todo-sync/notes --app todo + python3 -m tool.e2e_cleanup todo-sync/notes --app todo --apply +""" + +from __future__ import annotations + +import argparse +import sys +from typing import TYPE_CHECKING + +from crdt_sync import ( + PASSWORD_FILE, + E2eLockedError, + FirebaseConfig, + clear_plaintext_prefix, + firebase_client_for, +) + +if TYPE_CHECKING: + from crdt_sync import RemoteStore + + +def run(store: RemoteStore, old_prefix: str, passphrase: str, *, apply: bool) -> int: + """Clear (or list) ``old_prefix``; return a process exit code.""" + try: + paths = clear_plaintext_prefix( + store, old_prefix, passphrase=passphrase, apply=apply + ) + except E2eLockedError as exc: + sys.stdout.write(f"refused: {exc}\n") + return 1 + verb = "deleted" if apply else "would delete" + sys.stdout.write(f"{verb} {len(paths)} plaintext blob(s) under /{old_prefix}\n") + for path in paths: + sys.stdout.write(f" {path}\n") + return 0 + + +def main(argv: list[str]) -> int: # pragma: no cover - live database entry + """Parse args and run against the live database.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("old_prefix") + parser.add_argument("--app", required=True, help="seeded session name") + parser.add_argument("--apply", action="store_true") + args = parser.parse_args(argv) + client = firebase_client_for(args.app, config=FirebaseConfig.load()) + passphrase = PASSWORD_FILE.read_text(encoding="utf-8").strip() + return run(client, args.old_prefix, passphrase, apply=args.apply) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main(sys.argv[1:]))