mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-21 09:02:18 +01:00
import_media_archives.sh: fold Downloads media_archive zips into the cloud
Some checks are pending
Pre-commit checks / pre-commit (push) Waiting to run
Some checks are pending
Pre-commit checks / pre-commit (push) Waiting to run
New reusable utility that extracts each media_archive_*.zip snapshot, MOVEs its media into ~/cloud/Media/YYYY/MM via sync_media_to_cloud.sh (dedup across the overlapping snapshots), verifies staging drained, then deletes the zip — only after its media is confirmed in the cloud. Staging lives on the cloud filesystem so moves are atomic renames. Ran over ~/Downloads/media_archive_archive: 44 zips / 27 GB → 44 imported + deleted, 0 kept. The two 13.7 GB near-duplicate snapshots collapsed (2412 duplicates dropped). Cloud Media 4.9 GB → 19 GB (2949 files); Downloads reclaimed 27 GB; empty archive dir removed. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0126QvdMrv2WNeUVShFiLTc3
This commit is contained in:
parent
c6f1b85c78
commit
dfd0e2d70a
@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"title": "Import media_archive_*.zip snapshots into the dufs cloud and delete the zips",
|
||||||
|
"objective": "Give a one-command, safe, resumable way to fold the ~/Downloads/media_archive_archive snapshot zips into the self-hosted cloud: extract each, MOVE its media into ~/cloud/Media/YYYY/MM deduplicated against what is already there, and delete the zip only once its media is confirmed in the cloud — so Downloads reclaims the space with no data loss.",
|
||||||
|
"acceptance_criteria": [
|
||||||
|
"Each zip's media is moved into ~/cloud/Media/YYYY/MM by original mtime, deduplicated across the overlapping snapshots.",
|
||||||
|
"A zip is deleted only after its extracted media is verified to have left staging (moved or deduped); on any failure the zip is kept.",
|
||||||
|
"Staging is on the cloud filesystem so moves are atomic renames, not copies.",
|
||||||
|
"Only media is handled; the archives were verified to contain no non-media files, so nothing else is lost.",
|
||||||
|
"After the run the archive directory is empty and removed, and Downloads has reclaimed the space."
|
||||||
|
],
|
||||||
|
"out_of_scope": [
|
||||||
|
"Non-media file handling (none exist in these archives)",
|
||||||
|
"Full-content-hash dedup (organizer uses name+size within a date bucket)",
|
||||||
|
"Keeping the zips (explicitly deleted per user request)"
|
||||||
|
],
|
||||||
|
"verifier": "shellcheck; single-zip live test on a throwaway copy; full run over 44 zips (44 imported+deleted, 0 kept; 2412 duplicates dropped on the near-duplicate 13.7 GB snapshot; cloud Media 4.9->19 GB / 2949 files; archive dir removed); documented in docs/superpowers/evidence/import-media-archives-2026-07-12.json"
|
||||||
|
}
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"intent": "Extract the 44 media_archive_*.zip snapshots in ~/Downloads/media_archive_archive, MOVE all their media into the dufs cloud (deduplicated), and delete each zip after its media is safely in the cloud — reclaiming Downloads space.",
|
||||||
|
"scope": [
|
||||||
|
"New reusable tool: linux_configuration/scripts/single_use/utils/import_media_archives.sh",
|
||||||
|
"Reuses sync_media_to_cloud.sh (Phase-2 organizer) unchanged",
|
||||||
|
"Non-goal: no non-media handling (archives verified to contain only images/videos)"
|
||||||
|
],
|
||||||
|
"changes": [
|
||||||
|
"import_media_archives.sh: per-zip extract into a staging dir on the cloud filesystem (so mv is a rename), run sync_media_to_cloud.sh to MOVE media into Media/YYYY/MM with dedup, verify the staging drained, then delete the staging and the zip. A zip is deleted only after its media is confirmed in the cloud; any problem keeps the zip and continues.",
|
||||||
|
"Self-installs unzip on Arch; flock/dedup handled by the reused organizer."
|
||||||
|
],
|
||||||
|
"verification": [
|
||||||
|
{
|
||||||
|
"command": "shellcheck import_media_archives.sh",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "clean after fixing SC2155"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "single-zip live test on a throwaway copy",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "extract -> 1px.png moved to Media/2026/01 -> verify -> test zip deleted"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "full run over ~/Downloads/media_archive_archive (44 zips, 27 GB)",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "44 imported + deleted, 0 kept; the two 13.7 GB near-duplicate snapshots collapsed (zip 36 reported 2412 duplicates dropped); cloud Media 4.9 GB -> 19 GB (2949 files); Downloads archive dir emptied and removed; 27 GB reclaimed"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"risks": [
|
||||||
|
"Organizer dedup is by same-name+same-size within a YYYY/MM bucket (not full-hash); two genuinely different media with identical name+size+mtime would be treated as duplicates. Negligible for these dated snapshots.",
|
||||||
|
"Zips are permanently deleted after move — intended, and only after per-zip verification."
|
||||||
|
],
|
||||||
|
"rollback": [
|
||||||
|
"No rollback for deleted zips (by design). The media itself lives in ~/cloud/Media and is served by dufs; re-run generate_thumbnails.sh if previews are missing."
|
||||||
|
]
|
||||||
|
}
|
||||||
99
linux_configuration/scripts/single_use/utils/import_media_archives.sh
Executable file
99
linux_configuration/scripts/single_use/utils/import_media_archives.sh
Executable file
@ -0,0 +1,99 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# import_media_archives.sh — extract media_archive_*.zip snapshots and MOVE their
|
||||||
|
# media into the self-hosted dufs cloud, deduplicating across the overlapping
|
||||||
|
# snapshots, then delete each zip once its media is safely in the cloud.
|
||||||
|
#
|
||||||
|
# These archives are incremental snapshots of ~/Downloads (each holds a top-level
|
||||||
|
# downloads/ dir) and contain only images/videos — so "media only" loses nothing.
|
||||||
|
#
|
||||||
|
# Per zip (safe + resumable):
|
||||||
|
# 1. extract into a staging dir on the cloud filesystem (so mv is a rename)
|
||||||
|
# 2. hand the staging dir to sync_media_to_cloud.sh, which MOVES media into
|
||||||
|
# $CLOUD/Media/YYYY/MM (by mtime) and drops files already in the cloud
|
||||||
|
# 3. verify no media remains in staging (nothing failed to move)
|
||||||
|
# 4. delete the staging dir, then delete the zip
|
||||||
|
# A zip is deleted ONLY after its media is confirmed in the cloud; on any problem
|
||||||
|
# the zip is kept and the run continues.
|
||||||
|
#
|
||||||
|
# Usage: import_media_archives.sh [ARCHIVE_DIR]
|
||||||
|
# (default ARCHIVE_DIR: ~/Downloads/media_archive_archive)
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
HERE="$(dirname "$(readlink -f "$0")")"
|
||||||
|
readonly HERE
|
||||||
|
readonly ORGANIZER="$HERE/sync_media_to_cloud.sh"
|
||||||
|
readonly ARCHIVE_DIR="${1:-$HOME/Downloads/media_archive_archive}"
|
||||||
|
|
||||||
|
log() { printf '[import-archives] %s\n' "$*" >&2; }
|
||||||
|
die() { printf '[import-archives] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||||
|
|
||||||
|
# --- dependencies (self-installing on Arch) ----------------------------------
|
||||||
|
if ! command -v unzip >/dev/null; then
|
||||||
|
if command -v pacman >/dev/null; then
|
||||||
|
sudo pacman -S --needed --noconfirm unzip
|
||||||
|
else
|
||||||
|
die "unzip is required but not installed"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[[ -x "$ORGANIZER" ]] || die "organizer script not found/executable: $ORGANIZER"
|
||||||
|
[[ -d "$ARCHIVE_DIR" ]] || die "archive directory not found: $ARCHIVE_DIR"
|
||||||
|
|
||||||
|
# Media extensions must match sync_media_to_cloud.sh — used only to verify that
|
||||||
|
# staging was fully drained before deleting a zip.
|
||||||
|
readonly MEDIA_RE='.*\.(jpg|jpeg|png|gif|bmp|tiff|tif|webp|raw|cr2|nef|orf|arw|dng|heic|heif|mp4|avi|mkv|mov|wmv|flv|webm|m4v|3gp|ogv|mpg|mpeg|mts|m2ts|vob)$'
|
||||||
|
|
||||||
|
# --- stage on the cloud filesystem so mv is an atomic rename -----------------
|
||||||
|
CLOUD_ROOT="${CLOUD_ROOT:-}"
|
||||||
|
if [[ -z "$CLOUD_ROOT" && -f "$HOME/.config/dufs/dufs.yaml" ]]; then
|
||||||
|
CLOUD_ROOT="$(sed -nE 's/^serve-path:[[:space:]]*//p' "$HOME/.config/dufs/dufs.yaml" | head -1)"
|
||||||
|
fi
|
||||||
|
CLOUD_ROOT="${CLOUD_ROOT:-$HOME/cloud}"
|
||||||
|
STAGE_PARENT="$(dirname "$CLOUD_ROOT")"
|
||||||
|
readonly STAGE_PARENT
|
||||||
|
|
||||||
|
STAGING=""
|
||||||
|
cleanup() { [[ -n "$STAGING" && -d "$STAGING" ]] && rm -rf "$STAGING"; }
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
count_media() {
|
||||||
|
find "$1" -type f -regextype posix-extended -iregex "$MEDIA_RE" 2>/dev/null | wc -l
|
||||||
|
}
|
||||||
|
|
||||||
|
shopt -s nullglob
|
||||||
|
zips=("$ARCHIVE_DIR"/*.zip)
|
||||||
|
(( ${#zips[@]} )) || { log "no *.zip found in $ARCHIVE_DIR — nothing to do"; exit 0; }
|
||||||
|
|
||||||
|
readonly total=${#zips[@]}
|
||||||
|
i=0 deleted=0 kept=0
|
||||||
|
log "importing $total archive(s) from $ARCHIVE_DIR into $CLOUD_ROOT/Media"
|
||||||
|
for zip in "${zips[@]}"; do
|
||||||
|
i=$((i + 1))
|
||||||
|
log "[$i/$total] $(basename "$zip") ($(du -h "$zip" | cut -f1))"
|
||||||
|
|
||||||
|
STAGING="$(mktemp -d "$STAGE_PARENT/.media-import.XXXXXX")"
|
||||||
|
if ! unzip -q -o "$zip" -d "$STAGING"; then
|
||||||
|
log " unzip failed — keeping zip"; rm -rf "$STAGING"; STAGING=""; kept=$((kept + 1)); continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
before="$(count_media "$STAGING")"
|
||||||
|
if ! "$ORGANIZER" "$STAGING"; then
|
||||||
|
log " organizer failed — keeping zip"; rm -rf "$STAGING"; STAGING=""; kept=$((kept + 1)); continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
after="$(count_media "$STAGING")"
|
||||||
|
if (( after > 0 )); then
|
||||||
|
log " WARNING: $after/$before media file(s) not moved — keeping zip for safety"
|
||||||
|
rm -rf "$STAGING"; STAGING=""; kept=$((kept + 1)); continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -rf "$STAGING"; STAGING=""
|
||||||
|
if rm -f "$zip"; then
|
||||||
|
deleted=$((deleted + 1))
|
||||||
|
log " ✓ $before media handled; zip deleted"
|
||||||
|
else
|
||||||
|
log " media moved but zip delete failed: $zip"; kept=$((kept + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
log "done: $deleted zip(s) imported + deleted, $kept kept"
|
||||||
Loading…
Reference in New Issue
Block a user