From dfd0e2d70a6b621bf7ed2b659927b79b551ed62e Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 12 Jul 2026 17:43:23 +0200 Subject: [PATCH] import_media_archives.sh: fold Downloads media_archive zips into the cloud MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New reusable utility that extracts each media_archive_*.zip snapshot, MOVEs its media into ~/cloud/Media/YYYY/MM via sync_media_to_cloud.sh (dedup across the overlapping snapshots), verifies staging drained, then deletes the zip — only after its media is confirmed in the cloud. Staging lives on the cloud filesystem so moves are atomic renames. Ran over ~/Downloads/media_archive_archive: 44 zips / 27 GB → 44 imported + deleted, 0 kept. The two 13.7 GB near-duplicate snapshots collapsed (2412 duplicates dropped). Cloud Media 4.9 GB → 19 GB (2949 files); Downloads reclaimed 27 GB; empty archive dir removed. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_0126QvdMrv2WNeUVShFiLTc3 --- .../import-media-archives-2026-07-12.json | 17 ++++ .../import-media-archives-2026-07-12.json | 36 +++++++ .../single_use/utils/import_media_archives.sh | 99 +++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 docs/superpowers/contracts/import-media-archives-2026-07-12.json create mode 100644 docs/superpowers/evidence/import-media-archives-2026-07-12.json create mode 100755 linux_configuration/scripts/single_use/utils/import_media_archives.sh diff --git a/docs/superpowers/contracts/import-media-archives-2026-07-12.json b/docs/superpowers/contracts/import-media-archives-2026-07-12.json new file mode 100644 index 00000000..8b1a411f --- /dev/null +++ b/docs/superpowers/contracts/import-media-archives-2026-07-12.json @@ -0,0 +1,17 @@ +{ + "title": "Import media_archive_*.zip snapshots into the dufs cloud and delete the zips", + "objective": "Give a one-command, safe, resumable way to fold the ~/Downloads/media_archive_archive snapshot zips into the self-hosted cloud: extract each, MOVE its media into ~/cloud/Media/YYYY/MM deduplicated against what is already there, and delete the zip only once its media is confirmed in the cloud — so Downloads reclaims the space with no data loss.", + "acceptance_criteria": [ + "Each zip's media is moved into ~/cloud/Media/YYYY/MM by original mtime, deduplicated across the overlapping snapshots.", + "A zip is deleted only after its extracted media is verified to have left staging (moved or deduped); on any failure the zip is kept.", + "Staging is on the cloud filesystem so moves are atomic renames, not copies.", + "Only media is handled; the archives were verified to contain no non-media files, so nothing else is lost.", + "After the run the archive directory is empty and removed, and Downloads has reclaimed the space." + ], + "out_of_scope": [ + "Non-media file handling (none exist in these archives)", + "Full-content-hash dedup (organizer uses name+size within a date bucket)", + "Keeping the zips (explicitly deleted per user request)" + ], + "verifier": "shellcheck; single-zip live test on a throwaway copy; full run over 44 zips (44 imported+deleted, 0 kept; 2412 duplicates dropped on the near-duplicate 13.7 GB snapshot; cloud Media 4.9->19 GB / 2949 files; archive dir removed); documented in docs/superpowers/evidence/import-media-archives-2026-07-12.json" +} diff --git a/docs/superpowers/evidence/import-media-archives-2026-07-12.json b/docs/superpowers/evidence/import-media-archives-2026-07-12.json new file mode 100644 index 00000000..2175d3b7 --- /dev/null +++ b/docs/superpowers/evidence/import-media-archives-2026-07-12.json @@ -0,0 +1,36 @@ +{ + "intent": "Extract the 44 media_archive_*.zip snapshots in ~/Downloads/media_archive_archive, MOVE all their media into the dufs cloud (deduplicated), and delete each zip after its media is safely in the cloud — reclaiming Downloads space.", + "scope": [ + "New reusable tool: linux_configuration/scripts/single_use/utils/import_media_archives.sh", + "Reuses sync_media_to_cloud.sh (Phase-2 organizer) unchanged", + "Non-goal: no non-media handling (archives verified to contain only images/videos)" + ], + "changes": [ + "import_media_archives.sh: per-zip extract into a staging dir on the cloud filesystem (so mv is a rename), run sync_media_to_cloud.sh to MOVE media into Media/YYYY/MM with dedup, verify the staging drained, then delete the staging and the zip. A zip is deleted only after its media is confirmed in the cloud; any problem keeps the zip and continues.", + "Self-installs unzip on Arch; flock/dedup handled by the reused organizer." + ], + "verification": [ + { + "command": "shellcheck import_media_archives.sh", + "result": "pass", + "evidence": "clean after fixing SC2155" + }, + { + "command": "single-zip live test on a throwaway copy", + "result": "pass", + "evidence": "extract -> 1px.png moved to Media/2026/01 -> verify -> test zip deleted" + }, + { + "command": "full run over ~/Downloads/media_archive_archive (44 zips, 27 GB)", + "result": "pass", + "evidence": "44 imported + deleted, 0 kept; the two 13.7 GB near-duplicate snapshots collapsed (zip 36 reported 2412 duplicates dropped); cloud Media 4.9 GB -> 19 GB (2949 files); Downloads archive dir emptied and removed; 27 GB reclaimed" + } + ], + "risks": [ + "Organizer dedup is by same-name+same-size within a YYYY/MM bucket (not full-hash); two genuinely different media with identical name+size+mtime would be treated as duplicates. Negligible for these dated snapshots.", + "Zips are permanently deleted after move — intended, and only after per-zip verification." + ], + "rollback": [ + "No rollback for deleted zips (by design). The media itself lives in ~/cloud/Media and is served by dufs; re-run generate_thumbnails.sh if previews are missing." + ] +} diff --git a/linux_configuration/scripts/single_use/utils/import_media_archives.sh b/linux_configuration/scripts/single_use/utils/import_media_archives.sh new file mode 100755 index 00000000..f8a7a151 --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/import_media_archives.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# import_media_archives.sh — extract media_archive_*.zip snapshots and MOVE their +# media into the self-hosted dufs cloud, deduplicating across the overlapping +# snapshots, then delete each zip once its media is safely in the cloud. +# +# These archives are incremental snapshots of ~/Downloads (each holds a top-level +# downloads/ dir) and contain only images/videos — so "media only" loses nothing. +# +# Per zip (safe + resumable): +# 1. extract into a staging dir on the cloud filesystem (so mv is a rename) +# 2. hand the staging dir to sync_media_to_cloud.sh, which MOVES media into +# $CLOUD/Media/YYYY/MM (by mtime) and drops files already in the cloud +# 3. verify no media remains in staging (nothing failed to move) +# 4. delete the staging dir, then delete the zip +# A zip is deleted ONLY after its media is confirmed in the cloud; on any problem +# the zip is kept and the run continues. +# +# Usage: import_media_archives.sh [ARCHIVE_DIR] +# (default ARCHIVE_DIR: ~/Downloads/media_archive_archive) + +set -euo pipefail + +HERE="$(dirname "$(readlink -f "$0")")" +readonly HERE +readonly ORGANIZER="$HERE/sync_media_to_cloud.sh" +readonly ARCHIVE_DIR="${1:-$HOME/Downloads/media_archive_archive}" + +log() { printf '[import-archives] %s\n' "$*" >&2; } +die() { printf '[import-archives] ERROR: %s\n' "$*" >&2; exit 1; } + +# --- dependencies (self-installing on Arch) ---------------------------------- +if ! command -v unzip >/dev/null; then + if command -v pacman >/dev/null; then + sudo pacman -S --needed --noconfirm unzip + else + die "unzip is required but not installed" + fi +fi +[[ -x "$ORGANIZER" ]] || die "organizer script not found/executable: $ORGANIZER" +[[ -d "$ARCHIVE_DIR" ]] || die "archive directory not found: $ARCHIVE_DIR" + +# Media extensions must match sync_media_to_cloud.sh — used only to verify that +# staging was fully drained before deleting a zip. +readonly MEDIA_RE='.*\.(jpg|jpeg|png|gif|bmp|tiff|tif|webp|raw|cr2|nef|orf|arw|dng|heic|heif|mp4|avi|mkv|mov|wmv|flv|webm|m4v|3gp|ogv|mpg|mpeg|mts|m2ts|vob)$' + +# --- stage on the cloud filesystem so mv is an atomic rename ----------------- +CLOUD_ROOT="${CLOUD_ROOT:-}" +if [[ -z "$CLOUD_ROOT" && -f "$HOME/.config/dufs/dufs.yaml" ]]; then + CLOUD_ROOT="$(sed -nE 's/^serve-path:[[:space:]]*//p' "$HOME/.config/dufs/dufs.yaml" | head -1)" +fi +CLOUD_ROOT="${CLOUD_ROOT:-$HOME/cloud}" +STAGE_PARENT="$(dirname "$CLOUD_ROOT")" +readonly STAGE_PARENT + +STAGING="" +cleanup() { [[ -n "$STAGING" && -d "$STAGING" ]] && rm -rf "$STAGING"; } +trap cleanup EXIT + +count_media() { + find "$1" -type f -regextype posix-extended -iregex "$MEDIA_RE" 2>/dev/null | wc -l +} + +shopt -s nullglob +zips=("$ARCHIVE_DIR"/*.zip) +(( ${#zips[@]} )) || { log "no *.zip found in $ARCHIVE_DIR — nothing to do"; exit 0; } + +readonly total=${#zips[@]} +i=0 deleted=0 kept=0 +log "importing $total archive(s) from $ARCHIVE_DIR into $CLOUD_ROOT/Media" +for zip in "${zips[@]}"; do + i=$((i + 1)) + log "[$i/$total] $(basename "$zip") ($(du -h "$zip" | cut -f1))" + + STAGING="$(mktemp -d "$STAGE_PARENT/.media-import.XXXXXX")" + if ! unzip -q -o "$zip" -d "$STAGING"; then + log " unzip failed — keeping zip"; rm -rf "$STAGING"; STAGING=""; kept=$((kept + 1)); continue + fi + + before="$(count_media "$STAGING")" + if ! "$ORGANIZER" "$STAGING"; then + log " organizer failed — keeping zip"; rm -rf "$STAGING"; STAGING=""; kept=$((kept + 1)); continue + fi + + after="$(count_media "$STAGING")" + if (( after > 0 )); then + log " WARNING: $after/$before media file(s) not moved — keeping zip for safety" + rm -rf "$STAGING"; STAGING=""; kept=$((kept + 1)); continue + fi + + rm -rf "$STAGING"; STAGING="" + if rm -f "$zip"; then + deleted=$((deleted + 1)) + log " ✓ $before media handled; zip deleted" + else + log " media moved but zip delete failed: $zip"; kept=$((kept + 1)) + fi +done + +log "done: $deleted zip(s) imported + deleted, $kept kept"