mirror of
https://github.com/kuhyx/signal-bot.git
synced 2026-11-10 20:17:02 +01:00
tool/tagger.py did not run at all: 48957a1 renamed pack.default_dir to
default_dirs and missed the caller, so the tagger raised ImportError at
module scope through two releases. Nothing could have caught it -- tool/ is
outside --cov and mypy ran with [signal_bot, fake_signal]. Both are widened
here, and tests/test_tool_imports.py imports every script under tool/.
--import was a filename check and shutil.copy2. Pointed at a phone export
that overlaps the album it re-imported the same photo under whatever name
the exporter used that time. It now identifies files by magic bytes, strips
them through the same media_strip path !add_photo uses, and dedups on exact
bytes twice: the source hash, then -- only for what survives -- the hash
after stripping. sources.json records both, so a second run on the same
folder is a no-op. What the album refuses is moved to _skipped/<reason>/
beside the source, never deleted.
Two things that made "strip on import" impossible until now:
media_tags claims -all= zeroes the QuickTime date tags and allows them by
value. It does not; measured, it leaves CreateDate at its real value, and
every mp4 was being refused. -time:all= is what zeroes them.
And a tag allowlist cannot see the worst class at all. IMG_6924.mp4 carries
mebx timed-metadata holding per-frame face bounding boxes and a face id,
and those survive a complete tag strip because they live in mdat as opaque
samples. media_tracks.py drops the track with ffmpeg and verifies with
ffprobe that only video and audio survived. This is not the ffmpeg mistake
media_strip warns about: that was ffmpeg replacing exiftool for tags, this
is track removal with exiftool still doing every tag, and the gate is a
positive read-back rather than an exit code.
The roster was write-once. The setup form only appeared on an empty album,
Album.setup overwrote the file whole, and keys were positional (p{len+1}),
so inserting somebody renumbered everyone after them and repointed every
tags.json entry at the wrong person. Keys now come from the highest number
in use, the form merges, and a person can be added mid-session.
--untagged filters the page and never the server's list. /media/{index} and
/api/tag both index the full album, so a shorter list server-side would
show item i of the album while tagging item i of the filtered set.
Measured on the live album: 174 source files -> 1 not media, 82 already in
pack, 24 duplicate in batch, 67 imported, and re-running imports nothing.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBrE1b9HvFw3xvG9bJiDFt
134 lines
5.1 KiB
Python
134 lines
5.1 KiB
Python
# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence.
|
|
"""The import ledger: what the album has already taken in, and from what."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import TYPE_CHECKING
|
|
|
|
from signal_bot import pack_ledger
|
|
from signal_bot.pack import MEDIA_DIR
|
|
from signal_bot.pack_custom import FILE_MODE
|
|
from signal_bot.pack_ledger import LEDGER_FILE, Entry
|
|
from signal_bot.pack_store import ADDS_FILE
|
|
from tests.pack_fixture import PIXEL
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
def _album(tmp_path: Path, files: dict[str, bytes]) -> Path:
|
|
"""A pack directory holding exactly ``files`` in media/."""
|
|
media = tmp_path / MEDIA_DIR
|
|
media.mkdir(parents=True)
|
|
for name, payload in files.items():
|
|
(media / name).write_bytes(payload)
|
|
return tmp_path
|
|
|
|
|
|
def test_digest_is_a_full_sha256() -> None:
|
|
# Not content_name's truncated 32: a filename is a name, this is identity.
|
|
assert len(pack_ledger.digest(b"x")) == 64
|
|
|
|
|
|
def test_a_missing_ledger_reads_as_empty(tmp_path: Path) -> None:
|
|
assert pack_ledger.read(tmp_path) == {}
|
|
|
|
|
|
def test_a_corrupt_ledger_degrades_instead_of_raising(tmp_path: Path) -> None:
|
|
# The whole point: pack_load raises at startup and Restart=always turns
|
|
# that into a loop. A broken ledger must cost a duplicate, not the bot.
|
|
(tmp_path / LEDGER_FILE).write_text("{not json", encoding="utf-8")
|
|
assert pack_ledger.read(tmp_path) == {}
|
|
|
|
|
|
def test_a_ledger_that_is_not_an_object_reads_as_empty(tmp_path: Path) -> None:
|
|
(tmp_path / LEDGER_FILE).write_text("[1, 2]", encoding="utf-8")
|
|
assert pack_ledger.read(tmp_path) == {}
|
|
|
|
|
|
def test_one_malformed_row_does_not_lose_the_others(tmp_path: Path) -> None:
|
|
(tmp_path / LEDGER_FILE).write_text(
|
|
json.dumps(
|
|
{
|
|
"good.jpg": {"source": "a" * 64, "stored": "b" * 64},
|
|
"not-a-dict.jpg": "nonsense",
|
|
"no-stored.jpg": {"source": "c" * 64},
|
|
"bad-source.jpg": {"source": 7, "stored": "d" * 64},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
assert set(pack_ledger.read(tmp_path)) == {"good.jpg"}
|
|
|
|
|
|
def test_a_null_source_round_trips(tmp_path: Path) -> None:
|
|
entries = {"chat.jpg": Entry(source=None, stored="e" * 64)}
|
|
pack_ledger.write(tmp_path, entries)
|
|
assert pack_ledger.read(tmp_path) == entries
|
|
|
|
|
|
def test_the_ledger_is_written_private(tmp_path: Path) -> None:
|
|
pack_ledger.write(tmp_path, {"a.jpg": Entry(source="1" * 64, stored="2" * 64)})
|
|
assert (tmp_path / LEDGER_FILE).stat().st_mode & 0o777 == FILE_MODE
|
|
|
|
|
|
def test_source_hashes_skips_the_unrecoverable_ones() -> None:
|
|
entries = {
|
|
"a.jpg": Entry(source="1" * 64, stored="2" * 64),
|
|
"b.jpg": Entry(source=None, stored="3" * 64),
|
|
}
|
|
assert pack_ledger.source_hashes(entries) == {"1" * 64}
|
|
assert pack_ledger.stored_hashes(entries) == {"2" * 64, "3" * 64}
|
|
|
|
|
|
def test_backfill_records_an_imported_file_as_its_own_source(tmp_path: Path) -> None:
|
|
# shutil.copy2 copied these verbatim, so the bytes on disk *are* the
|
|
# source bytes. This is the only moment that is recoverable.
|
|
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
|
|
entries = pack_ledger.backfill(directory)
|
|
assert entries["IMG_1.jpg"].source == entries["IMG_1.jpg"].stored
|
|
assert entries["IMG_1.jpg"].stored == pack_ledger.digest(PIXEL)
|
|
|
|
|
|
def test_backfill_leaves_a_chat_upload_without_a_source(tmp_path: Path) -> None:
|
|
# adds.json is the exact record of what came in through !add_photo, and
|
|
# those were stripped on the way in: the source bytes never existed here.
|
|
directory = _album(tmp_path, {"abc123.jpg": PIXEL})
|
|
(directory / ADDS_FILE).write_text(
|
|
json.dumps({"abc123.jpg": {"by": "someone", "at": 1}}), encoding="utf-8"
|
|
)
|
|
entries = pack_ledger.backfill(directory)
|
|
assert entries["abc123.jpg"].source is None
|
|
assert entries["abc123.jpg"].stored == pack_ledger.digest(PIXEL)
|
|
|
|
|
|
def test_backfill_persists_what_it_found(tmp_path: Path) -> None:
|
|
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
|
|
pack_ledger.backfill(directory)
|
|
assert set(pack_ledger.read(directory)) == {"IMG_1.jpg"}
|
|
|
|
|
|
def test_backfill_never_revises_an_entry_it_already_has(tmp_path: Path) -> None:
|
|
# Re-running must not overwrite a source hash recorded before a strip
|
|
# with the post-strip bytes, which would silently lose the origin.
|
|
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
|
|
pack_ledger.write(directory, {"IMG_1.jpg": Entry(source="old", stored="older")})
|
|
entries = pack_ledger.backfill(directory)
|
|
assert entries["IMG_1.jpg"] == Entry(source="old", stored="older")
|
|
|
|
|
|
def test_backfill_is_idempotent(tmp_path: Path) -> None:
|
|
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
|
|
assert pack_ledger.backfill(directory) == pack_ledger.backfill(directory)
|
|
|
|
|
|
def test_backfill_ignores_a_directory_inside_media(tmp_path: Path) -> None:
|
|
directory = _album(tmp_path, {"IMG_1.jpg": PIXEL})
|
|
(directory / MEDIA_DIR / "subdir").mkdir()
|
|
assert set(pack_ledger.backfill(directory)) == {"IMG_1.jpg"}
|
|
|
|
|
|
def test_backfill_on_an_album_with_no_media_directory(tmp_path: Path) -> None:
|
|
assert pack_ledger.backfill(tmp_path) == {}
|