diff --git a/.gitignore b/.gitignore index 72b4de1..7c847e6 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ htmlcov/ screen_locker/log.json # ...and the ledger naming which of its entries record the same workout. screen_locker/duplicate_credits.json +# ...and the sync tombstones the PC re-pushes every tick (names a workout id). +screen_locker/sync_tombstones.json # ...including the timestamped backups a correction script leaves behind. screen_locker/*.json.bak-* screen_locker/sick_history.json diff --git a/screen_locker/_constants.py b/screen_locker/_constants.py index 475b804..2e19e6a 100644 --- a/screen_locker/_constants.py +++ b/screen_locker/_constants.py @@ -113,6 +113,9 @@ SCHEDULED_SKIPS_FILE = Path(__file__).resolve().parent / "scheduled_skips.json" EXTRA_BENEFITS_FILE = Path(__file__).resolve().parent / "extra_benefits_state.json" # State file storing the base (pre-bonus) shutdown hours and last reset date. SHUTDOWN_BASE_FILE = Path(__file__).resolve().parent / "shutdown_base.json" +# Workout records this PC must keep deleted in the sync store, pushed as +# tombstones on every tick (see _sync_tombstones for why once is not enough). +SYNC_TOMBSTONES_FILE = Path(__file__).resolve().parent / "sync_tombstones.json" # Self-expiring marker: "logged in during today's early-bird window, still # waiting to see if a real workout shows up." Not a log.json entry — # it's a same-day pending flag, checked against its own "date" field. diff --git a/screen_locker/_manual_push.py b/screen_locker/_manual_push.py index a3ffd7d..7ddf1cd 100644 --- a/screen_locker/_manual_push.py +++ b/screen_locker/_manual_push.py @@ -53,6 +53,7 @@ from screen_locker._log_io import load_workout_log from screen_locker._log_mixin import _derive_workout_id from screen_locker._push_outcome import describe_push from screen_locker._sync_retry import with_sync_retry +from screen_locker._sync_tombstones import tombstone_records from screen_locker._weekly_check import COUNTED_WORKOUT_TYPES from screen_locker._workout_sync import _DEVICES_PREFIX, read_sync_token, remote_client @@ -157,6 +158,9 @@ def records_from_workout_log(log_file: Path) -> dict[str, Record]: log[str(record_id)] = Record( id=str(record_id), fields={_PAYLOAD_FIELD: (payload, hlc)} ) + # Re-pushed every tick: sync_log never reads this device's own remote + # log, so a tombstone written once is overwritten by the next push. + log.update(tombstone_records()) return log diff --git a/screen_locker/_sync_tombstones.py b/screen_locker/_sync_tombstones.py new file mode 100644 index 0000000..8e9d150 --- /dev/null +++ b/screen_locker/_sync_tombstones.py @@ -0,0 +1,111 @@ +"""Durable tombstones for workout records the PC must keep deleted in sync. + +``_manual_push`` re-derives this device's sync log from ``log.json`` on every +tick and pushes it whole, and ``crdt_sync.sync_log`` pulls only the *other* +devices' logs — so a tombstone written once to the PC's device log is +overwritten by the very next push (observed 2026-09-13: ``--apply`` reported +"tombstoned", the dry run five seconds later reported "LIVE"). Deletion is +monotonic only for records that are still *in* the merge. + +The fix is to make the tombstone part of what the PC pushes every time. This +module owns that ledger: ``{record_id: {deleted_hlc, reason, tombstoned_at}}`` +in ``SYNC_TOMBSTONES_FILE``. The HLC is minted once and stored, so re-pushing +the same tombstone does not churn the store. + +The signed ``log.json`` is never touched — a tombstone only stops the record +from syncing; the local entry stays (see ``scripts/annotate_duplicate_credits.py`` +for why this repo annotates rather than deletes). +""" + +from __future__ import annotations + +from datetime import UTC, datetime +import json +import logging +from typing import TYPE_CHECKING, Any + +from crdt_sync import Hlc, Record + +from screen_locker._constants import SYNC_TOMBSTONES_FILE + +if TYPE_CHECKING: + from pathlib import Path + +_logger = logging.getLogger(__name__) + + +def _load(path: Path) -> dict[str, dict[str, Any]]: + """Read the ledger; a missing file is an empty ledger, a corrupt one is loud.""" + if not path.exists(): + return {} + try: + raw = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + _logger.warning( + "Sync tombstone ledger %s is unreadable (%s) — pushing NO tombstones " + "this tick, so a record deleted here may resurface on other devices", + path, + exc, + ) + return {} + if not isinstance(raw, dict): + _logger.warning( + "Sync tombstone ledger %s is not a JSON object — ignoring it", path + ) + return {} + return {str(rid): entry for rid, entry in raw.items() if isinstance(entry, dict)} + + +def _ledger_path(path: Path | None) -> Path: + """Resolve the ledger path at call time, so a patched module global applies. + + A default argument would bind the real file at import time and the test + suite's redirect (``tests/_isolated_state.py``) would silently miss it. + """ + return SYNC_TOMBSTONES_FILE if path is None else path + + +def add_tombstone( + record_id: str, + device_id: str, + reason: str, + path: Path | None = None, +) -> bool: + """Record *record_id* as deleted. Returns False if it already was.""" + path = _ledger_path(path) + ledger = _load(path) + if record_id in ledger: + return False + ledger[record_id] = { + "deleted_hlc": Hlc.new_tick(device_id).to_str(), + "reason": reason, + "tombstoned_at": datetime.now(tz=UTC).isoformat(), + } + path.write_text(json.dumps(ledger, indent=2, sort_keys=True) + "\n") + return True + + +def tombstone_records(path: Path | None = None) -> dict[str, Record]: + """Return every ledgered tombstone as a crdt_sync :class:`Record`. + + Entries without a parsable ``deleted_hlc`` are skipped with a warning + rather than pushed clockless: a tombstone with no HLC still deletes, but + it cannot be ordered against a later un-delete on another device. + """ + records: dict[str, Record] = {} + for record_id, entry in _load(_ledger_path(path)).items(): + hlc_str = entry.get("deleted_hlc") + try: + hlc = Hlc.from_str(str(hlc_str)) + except (ValueError, TypeError) as exc: + _logger.warning( + "Sync tombstone %s has no usable deleted_hlc (%r: %s) — not pushed", + record_id, + hlc_str, + exc, + ) + continue + records[record_id] = Record( + id=record_id, fields={}, deleted=True, deleted_hlc=hlc + ) + return records diff --git a/screen_locker/tests/_isolated_state.py b/screen_locker/tests/_isolated_state.py index 6e3aae2..d79b427 100644 --- a/screen_locker/tests/_isolated_state.py +++ b/screen_locker/tests/_isolated_state.py @@ -32,6 +32,15 @@ ISOLATED_STATE: tuple[tuple[str, tuple[str, ...]], ...] = ( "_status.EXTRA_BENEFITS_FILE", ), ), + # Pushed to the shared sync store on every tick; a test that wrote here + # would tombstone a real workout on the phone. + ( + "sync_tombstones.json", + ( + "_constants.SYNC_TOMBSTONES_FILE", + "_sync_tombstones.SYNC_TOMBSTONES_FILE", + ), + ), ( "sick_day_state.json", ( diff --git a/screen_locker/tests/test_sync_tombstones.py b/screen_locker/tests/test_sync_tombstones.py new file mode 100644 index 0000000..8c4ae08 --- /dev/null +++ b/screen_locker/tests/test_sync_tombstones.py @@ -0,0 +1,149 @@ +"""Tests for the durable sync-tombstone ledger and its place in every push. + +Guards the 2026-09-13 finding: a tombstone written once to the PC's device log +was gone by the next tick, because ``sync_log`` never reads this device's own +remote log and ``_manual_push`` re-derives it from ``log.json`` every time. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from crdt_sync import Hlc, Record + +from screen_locker import _sync_tombstones +from screen_locker._manual_push import records_from_workout_log +from screen_locker._sync_tombstones import add_tombstone, tombstone_records + +if TYPE_CHECKING: + from pathlib import Path + +_DEVICE = "57582b3b-01c6-45cc-9674-ad2b95bebf29" +_RID = "manual:2026-09-13T14:30" +_MANUAL = {"type": "manual_workout", "source": "football 11v11", "start_time": "14:30"} + + +class TestAddTombstone: + """The ledger is written once per id, with a clock minted at that moment.""" + + def test_creates_the_ledger_with_hlc_reason_and_time(self, tmp_path: Path) -> None: + path = tmp_path / "sync_tombstones.json" + assert add_tombstone(_RID, _DEVICE, "duplicate", path) is True + entry = json.loads(path.read_text())[_RID] + assert Hlc.from_str(entry["deleted_hlc"]).node_id == _DEVICE + assert entry["reason"] == "duplicate" + assert entry["tombstoned_at"].startswith("20") + + def test_second_add_is_a_no_op_and_keeps_the_original_clock( + self, tmp_path: Path + ) -> None: + path = tmp_path / "sync_tombstones.json" + add_tombstone(_RID, _DEVICE, "first", path) + before = path.read_text() + assert add_tombstone(_RID, _DEVICE, "second", path) is False + assert path.read_text() == before + + def test_adds_alongside_existing_ids(self, tmp_path: Path) -> None: + path = tmp_path / "sync_tombstones.json" + add_tombstone(_RID, _DEVICE, "a", path) + add_tombstone("manual:2026-01-01T08:00", _DEVICE, "b", path) + assert set(json.loads(path.read_text())) == {_RID, "manual:2026-01-01T08:00"} + + +class TestTombstoneRecords: + """Ledger → crdt_sync Records, loud about anything it cannot push.""" + + def test_missing_ledger_is_empty(self, tmp_path: Path) -> None: + assert tombstone_records(tmp_path / "absent.json") == {} + + def test_ledgered_id_becomes_a_deleted_record(self, tmp_path: Path) -> None: + path = tmp_path / "sync_tombstones.json" + add_tombstone(_RID, _DEVICE, "duplicate", path) + record = tombstone_records(path)[_RID] + assert record == Record( + id=_RID, + fields={}, + deleted=True, + deleted_hlc=Hlc.from_str(json.loads(path.read_text())[_RID]["deleted_hlc"]), + ) + + def test_same_clock_on_every_read(self, tmp_path: Path) -> None: + """Re-pushing must not churn the store: the HLC is stored, not re-minted.""" + path = tmp_path / "sync_tombstones.json" + add_tombstone(_RID, _DEVICE, "duplicate", path) + assert tombstone_records(path) == tombstone_records(path) + + def test_corrupt_ledger_pushes_nothing_and_warns( + self, tmp_path: Path, caplog + ) -> None: + path = tmp_path / "sync_tombstones.json" + path.write_text("{not json") + assert tombstone_records(path) == {} + assert "unreadable" in caplog.text + + def test_non_object_ledger_is_ignored_and_warns( + self, tmp_path: Path, caplog + ) -> None: + path = tmp_path / "sync_tombstones.json" + path.write_text("[]") + assert tombstone_records(path) == {} + assert "not a JSON object" in caplog.text + + def test_non_dict_entries_are_dropped(self, tmp_path: Path) -> None: + path = tmp_path / "sync_tombstones.json" + path.write_text(json.dumps({_RID: "garbage"})) + assert tombstone_records(path) == {} + + def test_entry_without_usable_hlc_is_skipped_and_warns( + self, tmp_path: Path, caplog + ) -> None: + path = tmp_path / "sync_tombstones.json" + path.write_text(json.dumps({_RID: {"reason": "x"}})) + assert tombstone_records(path) == {} + assert "no usable deleted_hlc" in caplog.text + + +class TestPushCarriesTombstones: + """records_from_workout_log replaces the live copy with the tombstone.""" + + def _log(self, tmp_path: Path) -> Path: + log_file = tmp_path / "log.json" + log_file.write_text( + json.dumps( + { + "2026-09-13": [ + { + "timestamp": "2026-09-13T07:40:06+00:00", + "workout_data": _MANUAL, + "workout_id": _RID, + } + ] + } + ) + ) + return log_file + + def test_live_entry_is_pushed_when_not_ledgered(self, tmp_path: Path) -> None: + log = records_from_workout_log(self._log(tmp_path)) + assert log[_RID].deleted is False + + def test_ledgered_entry_is_pushed_as_a_tombstone(self, tmp_path: Path) -> None: + """The isolated SYNC_TOMBSTONES_FILE (conftest) is what the push reads.""" + add_tombstone(_RID, _DEVICE, "duplicate", _sync_tombstones.SYNC_TOMBSTONES_FILE) + log = records_from_workout_log(self._log(tmp_path)) + assert log[_RID].deleted is True + assert log[_RID].fields == {} + + def test_tombstone_for_an_id_not_in_the_log_is_still_pushed( + self, tmp_path: Path + ) -> None: + add_tombstone( + "manual:2026-01-01T08:00", + _DEVICE, + "x", + _sync_tombstones.SYNC_TOMBSTONES_FILE, + ) + log = records_from_workout_log(self._log(tmp_path)) + assert log["manual:2026-01-01T08:00"].deleted is True + assert log[_RID].deleted is False diff --git a/scripts/tombstone_sync_record.py b/scripts/tombstone_sync_record.py new file mode 100755 index 0000000..b3e166a --- /dev/null +++ b/scripts/tombstone_sync_record.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Tombstone one workout record in the shared sync store, durably. + +Deletion in crdt_sync is monotonic: once a device log carries ``deleted=True`` +for an id, every merge keeps it deleted. But ``sync_log`` pulls only the OTHER +devices' logs and ``_manual_push`` re-derives the PC's log from ``log.json`` +every tick, so a tombstone written once to the PC's device log is gone by the +next push (seen 2026-09-13: "tombstoned", then "LIVE" five seconds later). +So this records the id in ``screen_locker/sync_tombstones.json`` +(``_sync_tombstones``), which every push carries, then runs one sync tick and +reports what the store holds afterwards. + +Written for ``manual:2026-09-13T14:30``: a synced copy of a football already +on disk, filed under a second id by that morning's day-key repair. It shares +the original's credit (``_weekly_check.credit_key``) and stays in the signed +log -- this repo annotates, never deletes -- but its sync record made every +15-minute tick log "budget exhausted -- not ingested". + +Dry-run by default: pulls the store and reports whether the id is live. +Rerunnable: an id already in the ledger is left as it is. + +Usage: + python3 scripts/tombstone_sync_record.py manual:2026-09-13T14:30 + python3 scripts/tombstone_sync_record.py manual:2026-09-13T14:30 --apply \ + --reason "synced duplicate of manual:2026-09-12T14:30" +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path +import sys + +from crdt_sync import ( + FileSyncStateStore, + GitHubSyncClient, + LogCodec, + Record, + RevisionTracking, + SyncTarget, + sync_log, +) + +from screen_locker._constants import ( + SYNC_REPO_NAME, + SYNC_REPO_OWNER, + SYNC_STATE_FILE, + SYNC_TIMEOUT_SECONDS, +) +from screen_locker._device import device_identity +from screen_locker._manual_push import ( + _decode_log, + _encode_log, + records_from_workout_log, +) +from screen_locker._sync_tombstones import add_tombstone +from screen_locker._workout_sync import _DEVICES_PREFIX, read_sync_token, remote_client + +_logger = logging.getLogger(__name__) + +# The same path ScreenLocker resolves at runtime (screen_lock.py), as the +# sibling scripts do; there is no package constant for it. +LOG_FILE = Path(__file__).resolve().parent.parent / "screen_locker" / "log.json" + + +def _report(message: str) -> None: + """Write one line to stdout (matches the other scripts in this directory).""" + sys.stdout.write(f"{message}\n") + + +def _sync(local_log: dict[str, Record]) -> dict[str, Record]: + """Run one sync tick with *local_log* as this device's view; return the merge.""" + identity = device_identity() + token = read_sync_token() + if token is None: + msg = "no sync token — cannot reach the store" + raise SystemExit(msg) + client = remote_client( + GitHubSyncClient( + SYNC_REPO_OWNER, SYNC_REPO_NAME, token, timeout_seconds=SYNC_TIMEOUT_SECONDS + ) + ) + return sync_log( + SyncTarget( + client=client, + device_id=identity.device_id, + legacy_device_id=identity.legacy_id, + path_prefix=_DEVICES_PREFIX, + ), + local_log, + LogCodec(decode=_decode_log, encode=_encode_log), + RevisionTracking(state_store=FileSyncStateStore(SYNC_STATE_FILE)), + ) + + +def main() -> int: + """Entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "record_id", help="sync record id, e.g. manual:2026-09-13T14:30" + ) + parser.add_argument("--apply", action="store_true", help="write the tombstone") + parser.add_argument("--reason", default="", help="why (kept in the ledger)") + args = parser.parse_args() + logging.basicConfig( + level=logging.WARNING, format="%(levelname)s %(name)s: %(message)s" + ) + + if args.apply: + if not args.reason: + parser.error("--apply needs --reason so the ledger says why") + if add_tombstone(args.record_id, device_identity().device_id, args.reason): + _report(f"{args.record_id}: added to the tombstone ledger") + else: + _report(f"{args.record_id}: already in the tombstone ledger") + # Exactly what _manual_push pushes on a tick: log.json plus the ledger. + local_log = records_from_workout_log(LOG_FILE) + merged = _sync(local_log) + + record = merged.get(args.record_id) + if record is None: + _report( + f"{args.record_id}: not in the store on any device — nothing to tombstone" + ) + return 1 + if record.deleted: + verb = "tombstoned" if args.apply else "already tombstoned" + _report(f"{args.record_id}: {verb} (deleted_hlc={record.deleted_hlc})") + return 0 + _report( + f"{args.record_id}: LIVE in the store — re-run with --apply to tombstone it" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())