# Copyright (c) 2026 Krzysztof Rudnicki. MIT licence. """Importing a folder of photos into the album without importing it twice. ``./web.sh --import`` used to be a filename check and ``shutil.copy2``. Pointed at a phone export that overlaps what is already tagged, that copies the same photo in again under whatever name the exporter gave it that time, and copies the metadata in with it. Both are fixed here: * **What a file is comes from its bytes** (media_sniff), not its suffix, the same rule ``!add_photo`` already follows. A suffix that disagrees with the bytes is replaced, because ``pack.media_type`` looks the stored suffix up in a closed table and a miss there fails the *next* load, not this import. * **Metadata is stripped on the way in**, through the same media_strip stripper the chat path uses, so the album has one rule instead of two. Dedup is exact bytes, twice, in this order: the source hash against everything the ledger has ever taken in, then -- only for what survives -- the stripped hash against what is on disk. The second stage is what makes a photo uploaded from the chat and the same photo in an export folder collapse into one, even though their source bytes differ. Whether a repeat is called "already in pack" or "duplicate in batch" is decided by *when* the hash was first seen: in the ledger before the run started, or earlier in this same run. That is the only difference between the two, and it is what lets the report's arithmetic be read as a funnel. Nothing is deleted. A file the album refuses is *moved* to a folder beside the source, one directory per reason, so a wrong call is visible and reversible rather than gone. """ from __future__ import annotations import shutil from dataclasses import dataclass, field from typing import TYPE_CHECKING from signal_bot import pack_ledger from signal_bot.media_sniff import sniff from signal_bot.pack import MEDIA_DIR, MEDIA_TYPES from signal_bot.pack_custom import FILE_MODE from signal_bot.pack_ledger import Entry from signal_bot.pack_report import SKIPPED_DIR, ImportReport, Outcome if TYPE_CHECKING: from collections.abc import Iterator from pathlib import Path from signal_bot.media_sniff import MediaKind from signal_bot.media_strip import Stripper @dataclass(frozen=True, slots=True) class Stored: """Everything an accepted file needs, computed once during planning. Carried so the apply pass never strips a second time: exiftool is the slow part of an import, and stripping twice would also mean trusting it to be byte-deterministic, which is exactly what the ledger exists to avoid depending on. """ name: str payload: bytes entry: Entry @dataclass(frozen=True, slots=True) class Decision: """What one source file is, and what the album will do with it.""" source: Path outcome: Outcome # Set for IMPORTED and for nothing else, which is what the apply pass # branches on -- so there is no state where the two disagree. stored: Stored | None = None @dataclass class _Seen: """Hashes and names already spoken for, split by when they were seen.""" ledger_source: set[str] ledger_stored: set[str] taken: set[str] batch_source: set[str] = field(default_factory=set) batch_stored: set[str] = field(default_factory=set) def plan_import(source: Path, directory: Path, stripper: Stripper) -> ImportReport: """Decide what would happen, without storing or moving anything. Not quite side-effect free: it backfills the ledger, which records what the album already holds. No photo is written, moved or altered. Args: source: The folder to import from. directory: The album directory. stripper: How metadata is removed; injected so tests never shell out. Returns: A report whose buckets account for every file in ``source``. """ # Backfilled here rather than behind a flag somebody has to remember: # an empty ledger makes every photo already in the album look new, and # the failure is a report that says "173 imported" instead of "85". entries = pack_ledger.backfill(directory) album = directory / MEDIA_DIR seen = _Seen( ledger_source=pack_ledger.source_hashes(entries), ledger_stored=pack_ledger.stored_hashes(entries), taken={path.name for path in album.iterdir()} if album.is_dir() else set(), ) before = len(seen.taken) decisions = tuple(_decide(path, stripper, seen) for path in _candidates(source)) return ImportReport(source=source, directory=directory, decisions=decisions, before=before) def run_import(source: Path, directory: Path, stripper: Stripper) -> ImportReport: """Import a folder: store what is new, move aside what is not. Args: source: The folder to import from. directory: The album directory. stripper: How metadata is removed. Returns: The report describing what was just done. A decision the recheck below overturns is reported as ALREADY_IN_PACK, not as imported. """ report = plan_import(source, directory, stripper) album = directory / MEDIA_DIR album.mkdir(parents=True, exist_ok=True) # Re-read rather than reuse the planning copy: planning backfilled, and a # photo may have arrived from the chat in between. entries = pack_ledger.backfill(directory) known = pack_ledger.stored_hashes(entries) settled: list[Decision] = [] for decision in report.decisions: if decision.stored is None: _reject(source, decision) settled.append(decision) continue if decision.stored.entry.stored in known: # Planning decided this was new; somebody uploaded it from the # chat while the strip pass was still running. Measured on # 2026-08-31: 6 photos landed that way during one import, and # storing them anyway put each in the album twice -- once # hash-named by !add_photo and once under its original name. _reject(source, Decision(decision.source, Outcome.ALREADY_IN_PACK)) settled.append(Decision(decision.source, Outcome.ALREADY_IN_PACK)) continue _store(album, decision.stored) entries[decision.stored.name] = decision.stored.entry known.add(decision.stored.entry.stored) settled.append(decision) pack_ledger.write(directory, entries) return ImportReport( source=report.source, directory=report.directory, decisions=tuple(settled), before=report.before, ) def _candidates(source: Path) -> Iterator[Path]: """Every regular file in the source folder, in a stable order.""" return (path for path in sorted(source.iterdir()) if path.is_file()) def _decide(path: Path, stripper: Stripper, seen: _Seen) -> Decision: """Bucket one file, claiming its hashes and name in ``seen`` as it goes.""" payload = path.read_bytes() kind = sniff(payload) if kind is None: return Decision(path, Outcome.NOT_MEDIA) source_hash = pack_ledger.digest(payload) repeat = _repeat(source_hash, seen.batch_source, seen.ledger_source) if repeat is not None: return Decision(path, repeat) cleaned = stripper.strip(payload, kind) if cleaned is None: return Decision(path, Outcome.UNSTRIPPABLE) stored_hash = pack_ledger.digest(cleaned) seen.batch_source.add(source_hash) repeat = _repeat(stored_hash, seen.batch_stored, seen.ledger_stored) if repeat is not None: return Decision(path, repeat) seen.batch_stored.add(stored_hash) name = _stored_name(path, kind, seen.taken) seen.taken.add(name) return Decision( path, Outcome.IMPORTED, stored=Stored( name=name, payload=cleaned, entry=Entry(source=source_hash, stored=stored_hash) ), ) def _repeat(digest: str, batch: set[str], ledger: set[str]) -> Outcome | None: """Which kind of repeat this hash is, or None when it is new. The only thing separating the two is *when* the hash was first seen: in the ledger before the run started, or earlier in this same run. """ if digest in batch: return Outcome.EXACT_DUP if digest in ledger: return Outcome.ALREADY_IN_PACK return None def _stored_name(path: Path, kind: MediaKind, taken: set[str]) -> str: """The name this file takes in media/: original, made safe and unique. The suffix is replaced when it disagrees with the bytes, because ``pack.media_type`` looks it up in a closed table and a miss there fails the next load. A name already in use by *different* bytes is disambiguated rather than skipped -- skipping is the old bug, and it lost a photo silently every time two exports both said "IMG_0001.jpg". """ suffix = path.suffix if MEDIA_TYPES.get(path.suffix.lower()) == kind.media_type else kind.suffix candidate = f"{path.stem}{suffix}" counter = 1 while candidate in taken: counter += 1 candidate = f"{path.stem}-{counter}{suffix}" return candidate def _store(album: Path, stored: Stored) -> None: """Write one accepted file into media/, atomically.""" path = album / stored.name temporary = path.with_name(f"{stored.name}.tmp") temporary.write_bytes(stored.payload) temporary.chmod(FILE_MODE) temporary.replace(path) def _reject(source: Path, decision: Decision) -> None: """Move one refused file into a folder beside the source, by reason.""" target = source.parent / SKIPPED_DIR / str(decision.outcome) target.mkdir(parents=True, exist_ok=True) shutil.move(str(decision.source), str(target / decision.source.name))