signal-bot/tests/test_pack_ledger.py
Krzysztof kuhy Rudnicki 9d546fef9a feat: import a folder of photos without importing it twice
tool/tagger.py did not run at all: 48957a1 renamed pack.default_dir to
default_dirs and missed the caller, so the tagger raised ImportError at
module scope through two releases. Nothing could have caught it -- tool/ is
outside --cov and mypy ran with [signal_bot, fake_signal]. Both are widened
here, and tests/test_tool_imports.py imports every script under tool/.

--import was a filename check and shutil.copy2. Pointed at a phone export
that overlaps the album it re-imported the same photo under whatever name
the exporter used that time. It now identifies files by magic bytes, strips
them through the same media_strip path !add_photo uses, and dedups on exact
bytes twice: the source hash, then -- only for what survives -- the hash
after stripping. sources.json records both, so a second run on the same
folder is a no-op. What the album refuses is moved to _skipped/<reason>/
beside the source, never deleted.

Two things that made "strip on import" impossible until now:

media_tags claims -all= zeroes the QuickTime date tags and allows them by
value. It does not; measured, it leaves CreateDate at its real value, and
every mp4 was being refused. -time:all= is what zeroes them.

And a tag allowlist cannot see the worst class at all. IMG_6924.mp4 carries
mebx timed-metadata holding per-frame face bounding boxes and a face id,
and those survive a complete tag strip because they live in mdat as opaque
samples. media_tracks.py drops the track with ffmpeg and verifies with
ffprobe that only video and audio survived. This is not the ffmpeg mistake
media_strip warns about: that was ffmpeg replacing exiftool for tags, this
is track removal with exiftool still doing every tag, and the gate is a
positive read-back rather than an exit code.

The roster was write-once. The setup form only appeared on an empty album,
Album.setup overwrote the file whole, and keys were positional (p{len+1}),
so inserting somebody renumbered everyone after them and repointed every
tags.json entry at the wrong person. Keys now come from the highest number
in use, the form merges, and a person can be added mid-session.

--untagged filters the page and never the server's list. /media/{index} and
/api/tag both index the full album, so a shorter list server-side would
show item i of the album while tagging item i of the filtered set.

Measured on the live album: 174 source files -> 1 not media, 82 already in
pack, 24 duplicate in batch, 67 imported, and re-running imports nothing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBrE1b9HvFw3xvG9bJiDFt
2026-08-31 13:22:57 +02:00

134 lines
5.1 KiB
Python

# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
"""The import ledger: what the album has already taken in, and from what."""
from __future__ import annotations
import json
from typing import TYPE_CHECKING
from signal_bot import pack_ledger
from signal_bot.pack import MEDIA_DIR
from signal_bot.pack_custom import FILE_MODE
from signal_bot.pack_ledger import LEDGER_FILE, Entry
from signal_bot.pack_store import ADDS_FILE
from tests.pack_fixture import PIXEL
if TYPE_CHECKING:
from pathlib import Path
def _album(tmp_path: Path, files: dict[str, bytes]) -> Path:
"""A pack directory holding exactly ``files`` in media/."""
media = tmp_path / MEDIA_DIR
media.mkdir(parents=True)
for name, payload in files.items():
(media / name).write_bytes(payload)
return tmp_path
def test_digest_is_a_full_sha256() -> None:
# Not content_name's truncated 32: a filename is a name, this is identity.
assert len(pack_ledger.digest(b"x")) == 64
def test_a_missing_ledger_reads_as_empty(tmp_path: Path) -> None:
assert pack_ledger.read(tmp_path) == {}
def test_a_corrupt_ledger_degrades_instead_of_raising(tmp_path: Path) -> None:
# The whole point: pack_load raises at startup and Restart=always turns
# that into a loop. A broken ledger must cost a duplicate, not the bot.
(tmp_path / LEDGER_FILE).write_text("{not json", encoding="utf-8")
assert pack_ledger.read(tmp_path) == {}
def test_a_ledger_that_is_not_an_object_reads_as_empty(tmp_path: Path) -> None:
(tmp_path / LEDGER_FILE).write_text("[1, 2]", encoding="utf-8")
assert pack_ledger.read(tmp_path) == {}
def test_one_malformed_row_does_not_lose_the_others(tmp_path: Path) -> None:
(tmp_path / LEDGER_FILE).write_text(
json.dumps(
{
"good.jpg": {"source": "a" * 64, "stored": "b" * 64},
"not-a-dict.jpg": "nonsense",
"no-stored.jpg": {"source": "c" * 64},
"bad-source.jpg": {"source": 7, "stored": "d" * 64},
}
),
encoding="utf-8",
)
assert set(pack_ledger.read(tmp_path)) == {"good.jpg"}
def test_a_null_source_round_trips(tmp_path: Path) -> None:
entries = {"chat.jpg": Entry(source=None, stored="e" * 64)}
pack_ledger.write(tmp_path, entries)
assert pack_ledger.read(tmp_path) == entries
def test_the_ledger_is_written_private(tmp_path: Path) -> None:
pack_ledger.write(tmp_path, {"a.jpg": Entry(source="1" * 64, stored="2" * 64)})
assert (tmp_path / LEDGER_FILE).stat().st_mode & 0o777 == FILE_MODE
def test_source_hashes_skips_the_unrecoverable_ones() -> None:
entries = {
"a.jpg": Entry(source="1" * 64, stored="2" * 64),
"b.jpg": Entry(source=None, stored="3" * 64),
}
assert pack_ledger.source_hashes(entries) == {"1" * 64}
assert pack_ledger.stored_hashes(entries) == {"2" * 64, "3" * 64}
def test_backfill_records_an_imported_file_as_its_own_source(tmp_path: Path) -> None:
# shutil.copy2 copied these verbatim, so the bytes on disk *are* the
# source bytes. This is the only moment that is recoverable.
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
entries = pack_ledger.backfill(directory)
assert entries["IMG_1.jpg"].source == entries["IMG_1.jpg"].stored
assert entries["IMG_1.jpg"].stored == pack_ledger.digest(PIXEL)
def test_backfill_leaves_a_chat_upload_without_a_source(tmp_path: Path) -> None:
# adds.json is the exact record of what came in through !add_photo, and
# those were stripped on the way in: the source bytes never existed here.
directory = _album(tmp_path, {"abc123.jpg": PIXEL})
(directory / ADDS_FILE).write_text(
json.dumps({"abc123.jpg": {"by": "someone", "at": 1}}), encoding="utf-8"
)
entries = pack_ledger.backfill(directory)
assert entries["abc123.jpg"].source is None
assert entries["abc123.jpg"].stored == pack_ledger.digest(PIXEL)
def test_backfill_persists_what_it_found(tmp_path: Path) -> None:
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
pack_ledger.backfill(directory)
assert set(pack_ledger.read(directory)) == {"IMG_1.jpg"}
def test_backfill_never_revises_an_entry_it_already_has(tmp_path: Path) -> None:
# Re-running must not overwrite a source hash recorded before a strip
# with the post-strip bytes, which would silently lose the origin.
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
pack_ledger.write(directory, {"IMG_1.jpg": Entry(source="old", stored="older")})
entries = pack_ledger.backfill(directory)
assert entries["IMG_1.jpg"] == Entry(source="old", stored="older")
def test_backfill_is_idempotent(tmp_path: Path) -> None:
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
assert pack_ledger.backfill(directory) == pack_ledger.backfill(directory)
def test_backfill_ignores_a_directory_inside_media(tmp_path: Path) -> None:
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
(directory / MEDIA_DIR / "subdir").mkdir()
assert set(pack_ledger.backfill(directory)) == {"IMG_1.jpg"}
def test_backfill_on_an_album_with_no_media_directory(tmp_path: Path) -> None:
assert pack_ledger.backfill(tmp_path) == {}