Tombstone sync records durably: a ledger the PC re-pushes every tick

manual:2026-09-13T14:30 -- the synced copy of the football that this
morning's day-key rekey let back in under a second id -- made every
workout-sync tick log "Manual-workout budget exhausted -- not ingested".
Tombstoning it once did not stick: sync_log pulls only the OTHER devices'
logs and _manual_push re-derives the PC's log from log.json each tick, so
the next push overwrote the tombstone ("tombstoned", then "LIVE" five
seconds later). Deletion is monotonic only for records still in the merge.

- New _sync_tombstones: {id: {deleted_hlc, reason, tombstoned_at}} in
  screen_locker/sync_tombstones.json (gitignored with its siblings). The
  HLC is minted once and stored so re-pushing does not churn the store.
  The ledger path is resolved at call time, not as a default argument, so
  the test suite's redirect actually applies -- verified by hashing the
  real ledger before and after the run.
- _manual_push.records_from_workout_log merges the ledger over the live
  derivation, so every push carries the tombstones.
- scripts/tombstone_sync_record.py <id> [--apply --reason ...]: dry run
  reports live/tombstoned; --apply writes the ledger, runs one tick and
  reports what the store holds afterwards. Rerunnable.
- tests/_isolated_state.py redirects SYNC_TOMBSTONES_FILE (both bindings).

Verified: --apply -> "tombstoned"; a second ordinary tick -> "already
tombstoned" with the same HLC; workout-sync.service now logs "Sync record
manual:2026-09-13T14:30 is tombstoned -- skipping it" and the budget line
for that id is gone. log.json untouched (the local entry stays; it shares
the original's credit per duplicate_credits.json).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01C7vgacLhb4WJjxuVj2QLEP

.gitignore

@@ -19,6 +19,8 @@ htmlcov/
 screen_locker/log.json
 # ...and the ledger naming which of its entries record the same workout.
 screen_locker/duplicate_credits.json
+# ...and the sync tombstones the PC re-pushes every tick (names a workout id).
+screen_locker/sync_tombstones.json
 # ...including the timestamped backups a correction script leaves behind.
 screen_locker/*.json.bak-*
 screen_locker/sick_history.json

screen_locker/_constants.py

@@ -113,6 +113,9 @@ SCHEDULED_SKIPS_FILE = Path(__file__).resolve().parent / "scheduled_skips.json"
 EXTRA_BENEFITS_FILE = Path(__file__).resolve().parent / "extra_benefits_state.json"
 # State file storing the base (pre-bonus) shutdown hours and last reset date.
 SHUTDOWN_BASE_FILE = Path(__file__).resolve().parent / "shutdown_base.json"
+# Workout records this PC must keep deleted in the sync store, pushed as
+# tombstones on every tick (see _sync_tombstones for why once is not enough).
+SYNC_TOMBSTONES_FILE = Path(__file__).resolve().parent / "sync_tombstones.json"
 # Self-expiring marker: "logged in during today's early-bird window, still
 # waiting to see if a real workout shows up." Not a log.json entry —
 # it's a same-day pending flag, checked against its own "date" field.

screen_locker/_manual_push.py

@@ -53,6 +53,7 @@ from screen_locker._log_io import load_workout_log
 from screen_locker._log_mixin import _derive_workout_id
 from screen_locker._push_outcome import describe_push
 from screen_locker._sync_retry import with_sync_retry
+from screen_locker._sync_tombstones import tombstone_records
 from screen_locker._weekly_check import COUNTED_WORKOUT_TYPES
 from screen_locker._workout_sync import _DEVICES_PREFIX, read_sync_token, remote_client
 
@@ -157,6 +158,9 @@ def records_from_workout_log(log_file: Path) -> dict[str, Record]:
             log[str(record_id)] = Record(
                 id=str(record_id), fields={_PAYLOAD_FIELD: (payload, hlc)}
             )
+    # Re-pushed every tick: sync_log never reads this device's own remote
+    # log, so a tombstone written once is overwritten by the next push.
+    log.update(tombstone_records())
     return log
 
 

screen_locker/_sync_tombstones.py

@@ -0,0 +1,111 @@
+"""Durable tombstones for workout records the PC must keep deleted in sync.
+
+``_manual_push`` re-derives this device's sync log from ``log.json`` on every
+tick and pushes it whole, and ``crdt_sync.sync_log`` pulls only the *other*
+devices' logs — so a tombstone written once to the PC's device log is
+overwritten by the very next push (observed 2026-09-13: ``--apply`` reported
+"tombstoned", the dry run five seconds later reported "LIVE"). Deletion is
+monotonic only for records that are still *in* the merge.
+
+The fix is to make the tombstone part of what the PC pushes every time. This
+module owns that ledger: ``{record_id: {deleted_hlc, reason, tombstoned_at}}``
+in ``SYNC_TOMBSTONES_FILE``. The HLC is minted once and stored, so re-pushing
+the same tombstone does not churn the store.
+
+The signed ``log.json`` is never touched — a tombstone only stops the record
+from syncing; the local entry stays (see ``scripts/annotate_duplicate_credits.py``
+for why this repo annotates rather than deletes).
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+
+from crdt_sync import Hlc, Record
+
+from screen_locker._constants import SYNC_TOMBSTONES_FILE
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+_logger = logging.getLogger(__name__)
+
+
+def _load(path: Path) -> dict[str, dict[str, Any]]:
+    """Read the ledger; a missing file is an empty ledger, a corrupt one is loud."""
+    if not path.exists():
+        return {}
+    try:
+        raw = json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError) as exc:
+        _logger.warning(
+            "Sync tombstone ledger %s is unreadable (%s) — pushing NO tombstones "
+            "this tick, so a record deleted here may resurface on other devices",
+            path,
+            exc,
+        )
+        return {}
+    if not isinstance(raw, dict):
+        _logger.warning(
+            "Sync tombstone ledger %s is not a JSON object — ignoring it", path
+        )
+        return {}
+    return {str(rid): entry for rid, entry in raw.items() if isinstance(entry, dict)}
+
+
+def _ledger_path(path: Path | None) -> Path:
+    """Resolve the ledger path at call time, so a patched module global applies.
+
+    A default argument would bind the real file at import time and the test
+    suite's redirect (``tests/_isolated_state.py``) would silently miss it.
+    """
+    return SYNC_TOMBSTONES_FILE if path is None else path
+
+
+def add_tombstone(
+    record_id: str,
+    device_id: str,
+    reason: str,
+    path: Path | None = None,
+) -> bool:
+    """Record *record_id* as deleted. Returns False if it already was."""
+    path = _ledger_path(path)
+    ledger = _load(path)
+    if record_id in ledger:
+        return False
+    ledger[record_id] = {
+        "deleted_hlc": Hlc.new_tick(device_id).to_str(),
+        "reason": reason,
+        "tombstoned_at": datetime.now(tz=UTC).isoformat(),
+    }
+    path.write_text(json.dumps(ledger, indent=2, sort_keys=True) + "\n")
+    return True
+
+
+def tombstone_records(path: Path | None = None) -> dict[str, Record]:
+    """Return every ledgered tombstone as a crdt_sync :class:`Record`.
+
+    Entries without a parsable ``deleted_hlc`` are skipped with a warning
+    rather than pushed clockless: a tombstone with no HLC still deletes, but
+    it cannot be ordered against a later un-delete on another device.
+    """
+    records: dict[str, Record] = {}
+    for record_id, entry in _load(_ledger_path(path)).items():
+        hlc_str = entry.get("deleted_hlc")
+        try:
+            hlc = Hlc.from_str(str(hlc_str))
+        except (ValueError, TypeError) as exc:
+            _logger.warning(
+                "Sync tombstone %s has no usable deleted_hlc (%r: %s) — not pushed",
+                record_id,
+                hlc_str,
+                exc,
+            )
+            continue
+        records[record_id] = Record(
+            id=record_id, fields={}, deleted=True, deleted_hlc=hlc
+        )
+    return records

screen_locker/tests/_isolated_state.py

@@ -32,6 +32,15 @@ ISOLATED_STATE: tuple[tuple[str, tuple[str, ...]], ...] = (
             "_status.EXTRA_BENEFITS_FILE",
         ),
     ),
+    # Pushed to the shared sync store on every tick; a test that wrote here
+    # would tombstone a real workout on the phone.
+    (
+        "sync_tombstones.json",
+        (
+            "_constants.SYNC_TOMBSTONES_FILE",
+            "_sync_tombstones.SYNC_TOMBSTONES_FILE",
+        ),
+    ),
     (
         "sick_day_state.json",
         (

screen_locker/tests/test_sync_tombstones.py

@@ -0,0 +1,149 @@
+"""Tests for the durable sync-tombstone ledger and its place in every push.
+
+Guards the 2026-09-13 finding: a tombstone written once to the PC's device log
+was gone by the next tick, because ``sync_log`` never reads this device's own
+remote log and ``_manual_push`` re-derives it from ``log.json`` every time.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from crdt_sync import Hlc, Record
+
+from screen_locker import _sync_tombstones
+from screen_locker._manual_push import records_from_workout_log
+from screen_locker._sync_tombstones import add_tombstone, tombstone_records
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+_DEVICE = "57582b3b-01c6-45cc-9674-ad2b95bebf29"
+_RID = "manual:2026-09-13T14:30"
+_MANUAL = {"type": "manual_workout", "source": "football 11v11", "start_time": "14:30"}
+
+
+class TestAddTombstone:
+    """The ledger is written once per id, with a clock minted at that moment."""
+
+    def test_creates_the_ledger_with_hlc_reason_and_time(self, tmp_path: Path) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        assert add_tombstone(_RID, _DEVICE, "duplicate", path) is True
+        entry = json.loads(path.read_text())[_RID]
+        assert Hlc.from_str(entry["deleted_hlc"]).node_id == _DEVICE
+        assert entry["reason"] == "duplicate"
+        assert entry["tombstoned_at"].startswith("20")
+
+    def test_second_add_is_a_no_op_and_keeps_the_original_clock(
+        self, tmp_path: Path
+    ) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        add_tombstone(_RID, _DEVICE, "first", path)
+        before = path.read_text()
+        assert add_tombstone(_RID, _DEVICE, "second", path) is False
+        assert path.read_text() == before
+
+    def test_adds_alongside_existing_ids(self, tmp_path: Path) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        add_tombstone(_RID, _DEVICE, "a", path)
+        add_tombstone("manual:2026-01-01T08:00", _DEVICE, "b", path)
+        assert set(json.loads(path.read_text())) == {_RID, "manual:2026-01-01T08:00"}
+
+
+class TestTombstoneRecords:
+    """Ledger → crdt_sync Records, loud about anything it cannot push."""
+
+    def test_missing_ledger_is_empty(self, tmp_path: Path) -> None:
+        assert tombstone_records(tmp_path / "absent.json") == {}
+
+    def test_ledgered_id_becomes_a_deleted_record(self, tmp_path: Path) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        add_tombstone(_RID, _DEVICE, "duplicate", path)
+        record = tombstone_records(path)[_RID]
+        assert record == Record(
+            id=_RID,
+            fields={},
+            deleted=True,
+            deleted_hlc=Hlc.from_str(json.loads(path.read_text())[_RID]["deleted_hlc"]),
+        )
+
+    def test_same_clock_on_every_read(self, tmp_path: Path) -> None:
+        """Re-pushing must not churn the store: the HLC is stored, not re-minted."""
+        path = tmp_path / "sync_tombstones.json"
+        add_tombstone(_RID, _DEVICE, "duplicate", path)
+        assert tombstone_records(path) == tombstone_records(path)
+
+    def test_corrupt_ledger_pushes_nothing_and_warns(
+        self, tmp_path: Path, caplog
+    ) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        path.write_text("{not json")
+        assert tombstone_records(path) == {}
+        assert "unreadable" in caplog.text
+
+    def test_non_object_ledger_is_ignored_and_warns(
+        self, tmp_path: Path, caplog
+    ) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        path.write_text("[]")
+        assert tombstone_records(path) == {}
+        assert "not a JSON object" in caplog.text
+
+    def test_non_dict_entries_are_dropped(self, tmp_path: Path) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        path.write_text(json.dumps({_RID: "garbage"}))
+        assert tombstone_records(path) == {}
+
+    def test_entry_without_usable_hlc_is_skipped_and_warns(
+        self, tmp_path: Path, caplog
+    ) -> None:
+        path = tmp_path / "sync_tombstones.json"
+        path.write_text(json.dumps({_RID: {"reason": "x"}}))
+        assert tombstone_records(path) == {}
+        assert "no usable deleted_hlc" in caplog.text
+
+
+class TestPushCarriesTombstones:
+    """records_from_workout_log replaces the live copy with the tombstone."""
+
+    def _log(self, tmp_path: Path) -> Path:
+        log_file = tmp_path / "log.json"
+        log_file.write_text(
+            json.dumps(
+                {
+                    "2026-09-13": [
+                        {
+                            "timestamp": "2026-09-13T07:40:06+00:00",
+                            "workout_data": _MANUAL,
+                            "workout_id": _RID,
+                        }
+                    ]
+                }
+            )
+        )
+        return log_file
+
+    def test_live_entry_is_pushed_when_not_ledgered(self, tmp_path: Path) -> None:
+        log = records_from_workout_log(self._log(tmp_path))
+        assert log[_RID].deleted is False
+
+    def test_ledgered_entry_is_pushed_as_a_tombstone(self, tmp_path: Path) -> None:
+        """The isolated SYNC_TOMBSTONES_FILE (conftest) is what the push reads."""
+        add_tombstone(_RID, _DEVICE, "duplicate", _sync_tombstones.SYNC_TOMBSTONES_FILE)
+        log = records_from_workout_log(self._log(tmp_path))
+        assert log[_RID].deleted is True
+        assert log[_RID].fields == {}
+
+    def test_tombstone_for_an_id_not_in_the_log_is_still_pushed(
+        self, tmp_path: Path
+    ) -> None:
+        add_tombstone(
+            "manual:2026-01-01T08:00",
+            _DEVICE,
+            "x",
+            _sync_tombstones.SYNC_TOMBSTONES_FILE,
+        )
+        log = records_from_workout_log(self._log(tmp_path))
+        assert log["manual:2026-01-01T08:00"].deleted is True
+        assert log[_RID].deleted is False

scripts/tombstone_sync_record.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Tombstone one workout record in the shared sync store, durably.
+
+Deletion in crdt_sync is monotonic: once a device log carries ``deleted=True``
+for an id, every merge keeps it deleted. But ``sync_log`` pulls only the OTHER
+devices' logs and ``_manual_push`` re-derives the PC's log from ``log.json``
+every tick, so a tombstone written once to the PC's device log is gone by the
+next push (seen 2026-09-13: "tombstoned", then "LIVE" five seconds later).
+So this records the id in ``screen_locker/sync_tombstones.json``
+(``_sync_tombstones``), which every push carries, then runs one sync tick and
+reports what the store holds afterwards.
+
+Written for ``manual:2026-09-13T14:30``: a synced copy of a football already
+on disk, filed under a second id by that morning's day-key repair. It shares
+the original's credit (``_weekly_check.credit_key``) and stays in the signed
+log -- this repo annotates, never deletes -- but its sync record made every
+15-minute tick log "budget exhausted -- not ingested".
+
+Dry-run by default: pulls the store and reports whether the id is live.
+Rerunnable: an id already in the ledger is left as it is.
+
+Usage:
+    python3 scripts/tombstone_sync_record.py manual:2026-09-13T14:30
+    python3 scripts/tombstone_sync_record.py manual:2026-09-13T14:30 --apply \
+        --reason "synced duplicate of manual:2026-09-12T14:30"
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+from pathlib import Path
+import sys
+
+from crdt_sync import (
+    FileSyncStateStore,
+    GitHubSyncClient,
+    LogCodec,
+    Record,
+    RevisionTracking,
+    SyncTarget,
+    sync_log,
+)
+
+from screen_locker._constants import (
+    SYNC_REPO_NAME,
+    SYNC_REPO_OWNER,
+    SYNC_STATE_FILE,
+    SYNC_TIMEOUT_SECONDS,
+)
+from screen_locker._device import device_identity
+from screen_locker._manual_push import (
+    _decode_log,
+    _encode_log,
+    records_from_workout_log,
+)
+from screen_locker._sync_tombstones import add_tombstone
+from screen_locker._workout_sync import _DEVICES_PREFIX, read_sync_token, remote_client
+
+_logger = logging.getLogger(__name__)
+
+# The same path ScreenLocker resolves at runtime (screen_lock.py), as the
+# sibling scripts do; there is no package constant for it.
+LOG_FILE = Path(__file__).resolve().parent.parent / "screen_locker" / "log.json"
+
+
+def _report(message: str) -> None:
+    """Write one line to stdout (matches the other scripts in this directory)."""
+    sys.stdout.write(f"{message}\n")
+
+
+def _sync(local_log: dict[str, Record]) -> dict[str, Record]:
+    """Run one sync tick with *local_log* as this device's view; return the merge."""
+    identity = device_identity()
+    token = read_sync_token()
+    if token is None:
+        msg = "no sync token — cannot reach the store"
+        raise SystemExit(msg)
+    client = remote_client(
+        GitHubSyncClient(
+            SYNC_REPO_OWNER, SYNC_REPO_NAME, token, timeout_seconds=SYNC_TIMEOUT_SECONDS
+        )
+    )
+    return sync_log(
+        SyncTarget(
+            client=client,
+            device_id=identity.device_id,
+            legacy_device_id=identity.legacy_id,
+            path_prefix=_DEVICES_PREFIX,
+        ),
+        local_log,
+        LogCodec(decode=_decode_log, encode=_encode_log),
+        RevisionTracking(state_store=FileSyncStateStore(SYNC_STATE_FILE)),
+    )
+
+
+def main() -> int:
+    """Entry point."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "record_id", help="sync record id, e.g. manual:2026-09-13T14:30"
+    )
+    parser.add_argument("--apply", action="store_true", help="write the tombstone")
+    parser.add_argument("--reason", default="", help="why (kept in the ledger)")
+    args = parser.parse_args()
+    logging.basicConfig(
+        level=logging.WARNING, format="%(levelname)s %(name)s: %(message)s"
+    )
+
+    if args.apply:
+        if not args.reason:
+            parser.error("--apply needs --reason so the ledger says why")
+        if add_tombstone(args.record_id, device_identity().device_id, args.reason):
+            _report(f"{args.record_id}: added to the tombstone ledger")
+        else:
+            _report(f"{args.record_id}: already in the tombstone ledger")
+    # Exactly what _manual_push pushes on a tick: log.json plus the ledger.
+    local_log = records_from_workout_log(LOG_FILE)
+    merged = _sync(local_log)
+
+    record = merged.get(args.record_id)
+    if record is None:
+        _report(
+            f"{args.record_id}: not in the store on any device — nothing to tombstone"
+        )
+        return 1
+    if record.deleted:
+        verb = "tombstoned" if args.apply else "already tombstoned"
+        _report(f"{args.record_id}: {verb} (deleted_hlc={record.deleted_hlc})")
+        return 0
+    _report(
+        f"{args.record_id}: LIVE in the store — re-run with --apply to tombstone it"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())