crdt-sync 0.11.1: verified plaintext cleanup

clear_plaintext_prefix + tool/e2e_cleanup: dry run by default, never
creates a header, deletes the old plaintext prefix only when every old
record is in the encrypted copy.

Todo: fda71ed1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crdt-sync/crdt_sync/__init__.py

@@ -21,6 +21,7 @@ from crdt_sync._e2e_cipher import (
     open_log_text,
     seal_log_text,
 )
+from crdt_sync._e2e_cleanup import clear_plaintext_prefix, unverified_blobs
 from crdt_sync._e2e_key import (
     Argon2Params,
     E2eHeaderError,
@@ -113,6 +114,7 @@ __all__ = [
     "SyncState",
     "SyncStateStore",
     "SyncTarget",
+    "clear_plaintext_prefix",
     "copy_to_encrypted",
     "credential_store_for",
     "default_e2e_path",
@@ -134,5 +136,6 @@ __all__ = [
     "seal_log_text",
     "sync_log",
     "unlock_e2e",
+    "unverified_blobs",
     "write_log",
 ]

crdt-sync/crdt_sync/_e2e_cleanup.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2026 Krzysztof Rudnicki
+"""Clear an app's old plaintext prefix once its encrypted copy is proven.
+
+The migration never deletes the old prefix by itself: an old build may still
+write there. Run this once every device runs an encrypting build. It never
+creates a header (a typo'd passphrase must not lock the app for good) and
+deletes nothing unless every old blob is contained in the encrypted copy.
+"""
+
+from __future__ import annotations
+
+import hmac
+from typing import TYPE_CHECKING
+
+from crdt_sync._e2e_key import derive_key, read_header
+from crdt_sync._e2e_store import E2eLayout, E2eLockedError, EncryptingStore, _blobs
+from crdt_sync._log import merge_logs
+from crdt_sync._store import dump_log, load_log
+
+if TYPE_CHECKING:
+    from crdt_sync._e2e_key import E2eKey
+    from crdt_sync._remote import RemoteStore
+
+
+def unverified_blobs(
+    store: RemoteStore, key: E2eKey, old_prefix: str, layout: E2eLayout
+) -> list[str]:
+    """Return old blobs (relative paths) not fully contained in the copy."""
+    dest = EncryptingStore(store, key, layout)
+    missing: list[str] = []
+    for rel, text in sorted(_blobs(store, old_prefix).items()):
+        try:
+            source = load_log(text)
+            copied = dest.get_file_text(f"{layout.prefix}/{rel}")
+            current = load_log(copied) if copied is not None else None
+        except ValueError, KeyError, TypeError:
+            missing.append(rel)
+            continue
+        if not source:
+            continue
+        if current is None or dump_log(merge_logs(current, source)) != dump_log(
+            current
+        ):
+            missing.append(rel)
+    return missing
+
+
+def _existing_key(store: RemoteStore, layout: E2eLayout, passphrase: str) -> E2eKey:
+    header = read_header(store, layout.e2e_path)
+    if header is None:
+        msg = f"no e2e header at {layout.e2e_path}: nothing was migrated"
+        raise E2eLockedError(msg)
+    key = derive_key(passphrase, header.salt, header.params)
+    if not hmac.compare_digest(key.check_for(header.salt), header.check):
+        msg = "wrong crdt-sync passphrase"
+        raise E2eLockedError(msg)
+    return key
+
+
+def clear_plaintext_prefix(
+    store: RemoteStore, old_prefix: str, *, passphrase: str, apply: bool
+) -> list[str]:
+    """Return the old paths to delete; delete them only when ``apply``.
+
+    Raises:
+        E2eLockedError: No header, a wrong passphrase, or a copy that does
+            not yet hold every old record.
+    """
+    layout = E2eLayout.of(old_prefix)
+    key = _existing_key(store, layout, passphrase)
+    missing = unverified_blobs(store, key, old_prefix, layout)
+    if missing:
+        msg = f"copy not verified for {', '.join(missing)}"
+        raise E2eLockedError(msg)
+    paths = [f"{old_prefix}/{rel}" for rel in sorted(_blobs(store, old_prefix))]
+    if apply:
+        for path in paths:
+            store.delete_file(path, message="e2e: clear plaintext prefix")
+    return paths

crdt-sync/crdt_sync/tests/test_e2e_cleanup.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2026 Krzysztof Rudnicki
+"""Clearing an old plaintext prefix only after the encrypted copy is proven."""
+
+from __future__ import annotations
+
+import io
+import json
+
+import pytest
+
+from crdt_sync import (
+    Argon2Params,
+    E2eLayout,
+    E2eLockedError,
+    clear_plaintext_prefix,
+    copy_to_encrypted,
+    unlock_e2e,
+    unverified_blobs,
+)
+from crdt_sync.tests.conftest import FakeStore
+from tool import e2e_cleanup
+
+CHEAP = Argon2Params(8, 1, 1)
+PHRASE = "pw"
+WRONG = PHRASE + "x"
+OLD = "app/devices"
+LAYOUT = E2eLayout.of(OLD)
+
+
+def _log(title: str, stamp: str = "2026-01-01T00:00:00.000Z") -> str:
+    return json.dumps(
+        {
+            "n": {
+                "id": "n",
+                "fields": {"t": [title, f"{stamp}-0000-a"]},
+                "deleted": False,
+                "deleted_hlc": None,
+            }
+        }
+    )
+
+
+def _seeded() -> FakeStore:
+    remote = FakeStore({f"{OLD}/pc/log.json": _log("x"), "app/revs/pc": "r"})
+    key = unlock_e2e(
+        remote,
+        path_prefix=LAYOUT.prefix,
+        passphrase=PHRASE,
+        params=CHEAP,
+        e2e_path=LAYOUT.e2e_path,
+    )
+    copy_to_encrypted(remote, key, OLD, LAYOUT)
+    return remote
+
+
+def test_dry_run_lists_and_deletes_nothing() -> None:
+    remote = _seeded()
+    paths = clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False)
+    assert paths == [f"{OLD}/pc/log.json"]
+    assert f"{OLD}/pc/log.json" in remote.files
+
+
+def test_apply_deletes_only_after_verification() -> None:
+    remote = _seeded()
+    clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=True)
+    assert f"{OLD}/pc/log.json" not in remote.files
+    assert f"{LAYOUT.prefix}/pc/log.json" in remote.files
+
+
+def test_unverified_copy_refuses() -> None:
+    remote = _seeded()
+    remote.files[f"{OLD}/pc/log.json"] = _log("newer", "2026-02-01T00:00:00.000Z")
+    remote.files[f"{OLD}/bad/log.json"] = "{oops"
+    key = unlock_e2e(
+        remote, path_prefix=LAYOUT.prefix, passphrase=PHRASE, e2e_path=LAYOUT.e2e_path
+    )
+    assert unverified_blobs(remote, key, OLD, LAYOUT) == ["bad/log.json", "pc/log.json"]
+    with pytest.raises(E2eLockedError, match="not verified"):
+        clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=True)
+    assert f"{OLD}/pc/log.json" in remote.files
+
+
+def test_empty_logs_need_no_copy() -> None:
+    remote = _seeded()
+    remote.files[f"{OLD}/tab/log.json"] = "{}"
+    paths = clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False)
+    assert f"{OLD}/tab/log.json" in paths
+
+
+def test_never_creates_a_header_and_wrong_passphrase_locks() -> None:
+    remote = FakeStore({f"{OLD}/pc/log.json": _log("x")})
+    with pytest.raises(E2eLockedError, match="no e2e header"):
+        clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False)
+    assert LAYOUT.e2e_path not in remote.files
+    seeded = _seeded()
+    with pytest.raises(E2eLockedError):
+        clear_plaintext_prefix(seeded, OLD, passphrase=WRONG, apply=False)
+
+
+def test_tool_main(monkeypatch: pytest.MonkeyPatch) -> None:
+    remote = _seeded()
+    out = io.StringIO()
+    monkeypatch.setattr("sys.stdout", out)
+    assert e2e_cleanup.run(remote, OLD, PHRASE, apply=False) == 0
+    assert "would delete 1" in out.getvalue()
+    assert e2e_cleanup.run(remote, OLD, PHRASE, apply=True) == 0
+    assert "deleted 1" in out.getvalue()
+    assert e2e_cleanup.run(FakeStore(), OLD, PHRASE, apply=True) == 1

crdt-sync/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "crdt-sync"
-version = "0.11.0"
+version = "0.11.1"
 description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps"
 requires-python = ">=3.14"
 dependencies = [

crdt-sync/tool/e2e_cleanup.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2026 Krzysztof Rudnicki
+"""Clear an app's old plaintext RTDB prefix after the E2E copy is verified.
+
+Dry run by default; ``--apply`` deletes. Run only once every device runs an
+encrypting build::
+
+    python3 -m tool.e2e_cleanup todo-sync/notes --app todo
+    python3 -m tool.e2e_cleanup todo-sync/notes --app todo --apply
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import TYPE_CHECKING
+
+from crdt_sync import (
+    PASSWORD_FILE,
+    E2eLockedError,
+    FirebaseConfig,
+    clear_plaintext_prefix,
+    firebase_client_for,
+)
+
+if TYPE_CHECKING:
+    from crdt_sync import RemoteStore
+
+
+def run(store: RemoteStore, old_prefix: str, passphrase: str, *, apply: bool) -> int:
+    """Clear (or list) ``old_prefix``; return a process exit code."""
+    try:
+        paths = clear_plaintext_prefix(
+            store, old_prefix, passphrase=passphrase, apply=apply
+        )
+    except E2eLockedError as exc:
+        sys.stdout.write(f"refused: {exc}\n")
+        return 1
+    verb = "deleted" if apply else "would delete"
+    sys.stdout.write(f"{verb} {len(paths)} plaintext blob(s) under /{old_prefix}\n")
+    for path in paths:
+        sys.stdout.write(f"  {path}\n")
+    return 0
+
+
+def main(argv: list[str]) -> int:  # pragma: no cover - live database entry
+    """Parse args and run against the live database."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("old_prefix")
+    parser.add_argument("--app", required=True, help="seeded session name")
+    parser.add_argument("--apply", action="store_true")
+    args = parser.parse_args(argv)
+    client = firebase_client_for(args.app, config=FirebaseConfig.load())
+    passphrase = PASSWORD_FILE.read_text(encoding="utf-8").strip()
+    return run(client, args.old_prefix, passphrase, apply=args.apply)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main(sys.argv[1:]))