diff --git a/docs/superpowers/contracts/media-cloud-sync-2026-07-12.json b/docs/superpowers/contracts/media-cloud-sync-2026-07-12.json new file mode 100644 index 00000000..0ce8f293 --- /dev/null +++ b/docs/superpowers/contracts/media-cloud-sync-2026-07-12.json @@ -0,0 +1,17 @@ +{ + "title": "Downloads media → dufs cloud (moved, unzipped, browsable)", + "objective": "Organized images and videos from ~/Downloads automatically MOVE into the dufs cloud, unzipped and date-organized, viewable from anywhere, with no duplication (the file leaves Downloads). Success = a settled image/video in ~/Downloads relocates to ~/cloud/Media/YYYY/MM, disappears from Downloads, and opens via the cloud URL, with no data lost.", + "acceptance_criteria": [ + "sync_media_to_cloud.sh MOVES each Downloads image/video into $CLOUD_ROOT/Media/YYYY/MM by mtime via a crash-safe copy-verify-rename-then-remove-source; idempotent, collision-safe, de-duplicates identical files.", + "Files modified within the last 2 minutes (possible in-progress downloads) are skipped and left in Downloads.", + "CLOUD_ROOT is derived from the dufs serve-path; media_archive zips and non-media are excluded; too_big/ media is included.", + "A systemd timer + path watcher keep the cloud current; a drop-in makes media-organizer run After the cloud sync so media is moved before the organizer runs.", + "The moved files are reachable and viewable via the dufs cloud URL." + ], + "out_of_scope": [ + "Retention / auto-pruning of ~/cloud/Media", + "Changing organize_downloads.sh's zip-and-delete archival", + "The web gallery (Phase 3) and mobile app (Phase 4)" + ], + "verifier": "shellcheck -x + bash -n; systemd-analyze verify on the units; sync_media_to_cloud.sh 8-assertion suite on throwaway dirs; real end-to-end (PNG → ~/cloud/Media → https URL 200 image/png); documented in docs/superpowers/evidence/media-cloud-sync-2026-07-12.json" +} diff --git a/docs/superpowers/evidence/media-cloud-sync-2026-07-12.json b/docs/superpowers/evidence/media-cloud-sync-2026-07-12.json new file mode 100644 index 00000000..84738914 --- /dev/null +++ b/docs/superpowers/evidence/media-cloud-sync-2026-07-12.json @@ -0,0 +1,44 @@ +{ + "intent": "MOVE organized Downloads media (images + videos) into the dufs cloud, unzipped and date-organized, so they're viewable from anywhere (web + upcoming app) WITHOUT duplication — the file leaves ~/Downloads and lives only in the cloud folder (on local disk, served by dufs). The cloud folder becomes the single storage.", + "scope": [ + "linux_configuration/scripts/single_use/utils/sync_media_to_cloud.sh (safe move of media → $CLOUD_ROOT/Media/YYYY/MM)", + "linux_configuration/scripts/single_use/utils/setup_media_cloud_sync.sh (installer: units + ordering drop-in)", + "systemd media-cloud-sync.{service,timer,path} + media-organizer.service.d/10-after-cloud-sync.conf", + "Non-goals: no auto-prune/retention of ~/cloud/Media; organize_downloads.sh unchanged (it now finds nothing to zip, since media is moved out first)" + ], + "changes": [ + "sync_media_to_cloud.sh sweeps ~/Downloads (recursively, including too_big/) for the same image/video extensions organize_downloads uses and MOVES each into $CLOUD_ROOT/Media/YYYY/MM/ by file mtime. The move is crash-safe: copy to a .partial temp, verify size, atomic rename, then remove the source. Files modified within the last 2 minutes are skipped (possible in-progress downloads). A file identical to one already in the cloud is de-duplicated (source dropped); a name clash with a different file gets a numeric suffix. flock-guarded; excludes media_archive_*.zip and staging dirs. CLOUD_ROOT from the dufs serve-path (fallback ~/cloud).", + "setup_media_cloud_sync.sh installs a oneshot service + a 30-min catch-up timer + a ~/Downloads path watcher, plus a drop-in making media-organizer run After=media-cloud-sync so media is moved to the cloud before the organizer runs. Runs an initial sweep." + ], + "verification": [ + { + "command": "shellcheck -x --severity=warning + bash -n on both scripts; systemd-analyze verify on the three units", + "result": "pass", + "evidence": "both scripts clean; units structurally valid." + }, + { + "command": "sync_media_to_cloud.sh MOVE assertion suite (throwaway dirs)", + "result": "pass", + "evidence": "12/12: media MOVED (source removed) into Media/YYYY/MM by mtime; too_big included; media_archive zip and non-media left alone; a recently-modified file is SKIPPED and left in Downloads; idempotent re-run moves nothing; an identical file is de-duplicated (source dropped, no cloud duplicate); a clashing different file gets a suffix." + }, + { + "command": "Real end-to-end: MOVE media → cloud → live dufs URL", + "result": "pass", + "evidence": "a PNG moved from a temp source into /home/kuhy/cloud/Media/2026/05/ (source removed) and fetched via https://kuhy-cloud.duckdns.org/Media/2026/05/phase2_move.png → HTTP 200, image/png. Test file cleaned up." + }, + { + "command": "Installer real apply (systemd units) + full initial sweep", + "result": "pending", + "evidence": "Needs sudo, so run by the user via '! setup_media_cloud_sync.sh'. The initial sweep RELOCATES ~4.9 GB of current Downloads media into ~/cloud/Media (same disk — frees Downloads, no net duplication). To confirm." + } + ], + "risks": [ + "Move is destructive by design: media leaves ~/Downloads and exists only in ~/cloud/Media (on local disk, reachable via the auth-protected dufs cloud). Mitigated by crash-safe move (verify before removing source) and the 2-minute quiesce window for in-progress downloads.", + "No retention/auto-prune of ~/cloud/Media yet — it grows over time.", + "The path watcher fires on top-level ~/Downloads changes; deeply-nested new files are caught by the 30-min timer instead of instantly." + ], + "rollback": [ + "sudo systemctl disable --now media-cloud-sync.timer media-cloud-sync.path; sudo rm /etc/systemd/system/media-cloud-sync.{service,timer,path} /etc/systemd/system/media-organizer.service.d/10-after-cloud-sync.conf; sudo systemctl daemon-reload.", + "Media already moved lives under ~/cloud/Media; move any of it back to ~/Downloads if desired. Nothing is deleted by rollback." + ] +} diff --git a/linux_configuration/scripts/single_use/utils/setup_media_cloud_sync.sh b/linux_configuration/scripts/single_use/utils/setup_media_cloud_sync.sh new file mode 100755 index 00000000..60a18dcf --- /dev/null +++ b/linux_configuration/scripts/single_use/utils/setup_media_cloud_sync.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# setup_media_cloud_sync.sh — install the "downloads media → dufs cloud" sync. +# +# Installs systemd units that mirror ~/Downloads images/videos into the dufs +# cloud (unzipped + browsable) so they're viewable from anywhere: +# • media-cloud-sync.service (oneshot → sync_media_to_cloud.sh) +# • media-cloud-sync.timer (catch-up every 30 min) +# • media-cloud-sync.path (immediate, when ~/Downloads changes) +# Plus a drop-in so the existing media-organizer (which zips + deletes Downloads +# for local cold archive) runs AFTER the cloud sync — nothing is deleted before +# the cloud has a copy. +# +# Runs on the dufs host (the PC). Run as your normal user; sudo is used only for +# the system units. + +set -euo pipefail + +HERE="$(dirname "$(readlink -f "$0")")" +readonly HERE +readonly SYNC_SCRIPT="$HERE/sync_media_to_cloud.sh" +RUN_USER="$(id -un)" +readonly RUN_USER +readonly DOWNLOADS="$HOME/Downloads" + +C() { printf '\033[1;34m[media-cloud-setup]\033[0m %s\n' "$*"; } +OK() { printf '\033[1;32m ✓\033[0m %s\n' "$*"; } +die() { printf '\033[1;31m[media-cloud-setup] ERROR:\033[0m %s\n' "$*" >&2; exit 1; } +usage() { grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'; exit 0; } + +write_unit() { # write_unit + sudo tee "/etc/systemd/system/$1" >/dev/null + OK "wrote /etc/systemd/system/$1" +} + +main() { + [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]] && usage + [[ -x "$SYNC_SCRIPT" ]] || die "missing $SYNC_SCRIPT" + + # fd is the only extra dependency of the sync script. + if command -v pacman >/dev/null; then + sudo pacman -S --needed --noconfirm fd >/dev/null + elif command -v apt-get >/dev/null; then + sudo apt-get install -y fd-find >/dev/null + fi + command -v fd >/dev/null || command -v fdfind >/dev/null || die "fd (fd-find) not installed" + OK "dependencies present" + + C "Installing systemd units" + write_unit media-cloud-sync.service </dev/null </dev/null | head -1)" + cloud_root="${cloud_root:-$HOME/cloud}" + cat <&2; } + +readonly QUIESCE_SECONDS=120 # skip files modified within this window (still downloading) + +# --- where does the cloud live? ----------------------------------------------- +CLOUD_ROOT="${CLOUD_ROOT:-}" +if [[ -z "$CLOUD_ROOT" && -f "$HOME/.config/dufs/dufs.yaml" ]]; then + CLOUD_ROOT="$(sed -nE 's/^serve-path:[[:space:]]*//p' "$HOME/.config/dufs/dufs.yaml" | head -1)" +fi +CLOUD_ROOT="${CLOUD_ROOT:-$HOME/cloud}" +readonly MEDIA_DEST="$CLOUD_ROOT/Media" + +SOURCES=("$@") +[[ ${#SOURCES[@]} -eq 0 ]] && SOURCES=("$HOME/Downloads") + +FD="$(command -v fd || command -v fdfind || true)" +[[ -n "$FD" ]] || { log "ERROR: fd (fd-find) not installed"; exit 1; } + +# Same media extensions as organize_downloads.sh. +readonly EXTS=(jpg jpeg png gif bmp tiff tif webp raw cr2 nef orf arw dng heic heif \ + mp4 avi mkv mov wmv flv webm m4v 3gp ogv mpg mpeg mts m2ts vob) +FD_EXT_ARGS=() +for e in "${EXTS[@]}"; do FD_EXT_ARGS+=(-e "$e"); done + +# --- serialize --------------------------------------------------------------- +STATE_DIR="${MEDIA_SYNC_STATE:-$HOME/.local/state/media-cloud-sync}" +mkdir -p "$STATE_DIR" "$MEDIA_DEST" +exec 9>"$STATE_DIR/lock" +flock -n 9 || { log "another sync is running — skipping"; exit 0; } + +now="$(date +%s)" +moved=0 deduped=0 skipped=0 +for src in "${SOURCES[@]}"; do + [[ -e "$src" ]] || continue + if [[ -f "$src" ]]; then + mapfile -t FILES < <(printf '%s\n' "$src") + else + mapfile -t FILES < <( + "$FD" "${FD_EXT_ARGS[@]}" --type f \ + --exclude 'media_archive_*' --exclude '*media_organize_*' \ + --exclude '.git' --exclude 'node_modules' \ + . "$src" 2>/dev/null || true + ) + fi + for f in "${FILES[@]}"; do + [[ -f "$f" ]] || continue + # Skip files still being written (recently modified → maybe downloading). + mtime="$(stat -c%Y "$f" 2>/dev/null || echo 0)" + if (( now - mtime < QUIESCE_SECONDS )); then + log "skip (modified <${QUIESCE_SECONDS}s ago, may be downloading): $f" + skipped=$((skipped + 1)); continue + fi + ext="${f##*.}"; ext="${ext,,}" + sub="$(date -r "$f" +%Y/%m 2>/dev/null || echo unknown)" + dest_dir="$MEDIA_DEST/$sub" + base="$(basename "$f")" + dest="$dest_dir/$base" + if [[ -e "$dest" ]]; then + if [[ "$(stat -c%s "$f")" == "$(stat -c%s "$dest" 2>/dev/null)" ]]; then + rm -f "$f"; deduped=$((deduped + 1)); continue # identical already in cloud + fi + stem="${base%.*}"; n=1 + while [[ -e "$dest_dir/${stem}_${n}.${ext}" ]]; do n=$((n + 1)); done + dest="$dest_dir/${stem}_${n}.${ext}" + fi + mkdir -p "$dest_dir" + # $dest is a pre-checked free name, so mv won't clobber. mv is atomic on + # the same filesystem and safe (copy-then-unlink) across filesystems. + if mv -f "$f" "$dest" 2>/dev/null; then + moved=$((moved + 1)) + else + log "WARNING: failed to move $f (left in Downloads)" + fi + done +done + +log "done: $moved moved, $deduped duplicates dropped, $skipped skipped → $MEDIA_DEST"