From 33f0fee3424d337e7078ca9a525bbde3cdcfc447 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Mon, 28 Sep 2026 10:44:00 +0200 Subject: [PATCH] wsg_grabber: keep to the cloud folder, cap trash at 200 passes Kept videos land in ~/data/cloud/Media//wsg (~/Downloads without the cloud folder); trash/ keeps only the newest 200 passes and marks the rest purged. Legacy keep/ is migrated once. Undo follows a keep through every place it can end up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01F8G3BNETGmbr6aM5cc18g4 --- ...abber-cloud-keep-trash-cap-2026-09-18.json | 18 +++ ...erage-bar-back-to-presence-2026-08-23.json | 43 ------ ...abber-cloud-keep-trash-cap-2026-09-18.json | 46 +++++++ python_pkg/wsg_grabber/README.md | 48 ++++--- python_pkg/wsg_grabber/_migrate_keep.py | 74 ++++++++++ python_pkg/wsg_grabber/_prune.py | 56 ++++++++ python_pkg/wsg_grabber/app.py | 7 +- python_pkg/wsg_grabber/cli.py | 9 +- python_pkg/wsg_grabber/constants.py | 5 + python_pkg/wsg_grabber/files.py | 22 ++- python_pkg/wsg_grabber/housekeeping.py | 66 +++++++++ python_pkg/wsg_grabber/paths.py | 81 ++++++++++- python_pkg/wsg_grabber/states.py | 5 +- python_pkg/wsg_grabber/store_verdicts.py | 69 ++++++++++ python_pkg/wsg_grabber/tests/test_app_undo.py | 9 +- python_pkg/wsg_grabber/tests/test_cli.py | 7 +- .../wsg_grabber/tests/test_housekeeping.py | 92 +++++++++++++ .../wsg_grabber/tests/test_migrate_keep.py | 130 ++++++++++++++++++ python_pkg/wsg_grabber/tests/test_paths.py | 31 ++++- python_pkg/wsg_grabber/tests/test_prune.py | 127 +++++++++++++++++ python_pkg/wsg_grabber/tests/test_states.py | 9 +- 21 files changed, 871 insertions(+), 83 deletions(-) create mode 100644 docs/superpowers/contracts/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json delete mode 100644 docs/superpowers/evidence/shell-coverage-bar-back-to-presence-2026-08-23.json create mode 100644 docs/superpowers/evidence/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json create mode 100644 python_pkg/wsg_grabber/_migrate_keep.py create mode 100644 python_pkg/wsg_grabber/_prune.py create mode 100644 python_pkg/wsg_grabber/housekeeping.py create mode 100644 python_pkg/wsg_grabber/tests/test_housekeeping.py create mode 100644 python_pkg/wsg_grabber/tests/test_migrate_keep.py create mode 100644 python_pkg/wsg_grabber/tests/test_prune.py diff --git a/docs/superpowers/contracts/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json b/docs/superpowers/contracts/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json new file mode 100644 index 00000000..eb6216af --- /dev/null +++ b/docs/superpowers/contracts/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json @@ -0,0 +1,18 @@ +{ + "title": "wsg_grabber: keep to the cloud folder, cap trash at 200 passes", + "objective": "Kept videos land somewhere a person looks (~/data/cloud/Media//wsg, ~/Downloads without the cloud folder) instead of ~/.local/share/wsg_grabber/keep, and trash/ stops growing without bound: only the 200 most recent passes stay on disk and undoable, everything older is deleted for good.", + "acceptance_criteria": [ + "`wsg-grabber stats` prints `keep dir ~/data/cloud/Media//wsg`, and a keep verdict moves the file there.", + "With HOME pointed at a tree without data/cloud, a keep lands in $HOME/Downloads.", + "After startup and after every pass, `ls trash | wc -l` is at most 200 and the survivors are exactly the 200 newest reviewed_at PASSED rows; orphans without an index row are deleted too.", + "Rows for deleted trash files become the new terminal state `purged`: still in known_md5s (never re-downloaded), excluded from the undo trail.", + "The old keep/ contents are migrated on first start (same name+size at the destination drops the duplicate, other clashes get a -2 suffix and the index row is renamed), and the empty keep/ is removed.", + "Undo still restores a keep from wherever it ended up (this year's cloud folder, earlier years', ~/Downloads, the legacy keep/, Media//) and a pass from trash/." + ], + "out_of_scope": [ + "Capping kept files: they are the product and stay forever.", + "A schema column for the keep directory; the candidate search covers every location without one.", + "Any change to dufs-cloud, the gallery, or the media-cloud-sync timer." + ], + "verifier": "pytest python_pkg/wsg_grabber/tests (303 passed, every wsg_grabber module 100% branch coverage); rehearsal in a fabricated HOME with a copy of the real index and stand-in files; then the real run: keep/ 342 -> cloud 647, trash 23218 files / 77 GB -> 200 files / 823 MB, 23018 rows purged." +} diff --git a/docs/superpowers/evidence/shell-coverage-bar-back-to-presence-2026-08-23.json b/docs/superpowers/evidence/shell-coverage-bar-back-to-presence-2026-08-23.json deleted file mode 100644 index fe70f1fd..00000000 --- a/docs/superpowers/evidence/shell-coverage-bar-back-to-presence-2026-08-23.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "intent": "Stop enforcing a measured 100% line-coverage bar on shell libraries, which priced the remaining work at several days, and return the ratchet to the presence check it was before 2026-08-22.", - "scope": [ - "meta/scripts/lib/shell_coverage_bar.sh", - "Non-goal: deleting shell_coverage_jail.sh, which is still used deliberately and by ci_mirror.sh", - "Non-goal: changing python_pkg's 100% branch-coverage bar, which is already met" - ], - "changes": [ - "is_covered() is once again 'the lib's directory has a tests/run_all.sh', replacing the jail-measured COVERAGE_BAR=100 introduced in bca70b67.", - "Recorded why in the file: 22 libs in fixes/lib required 34 hand-written test files, the jail runs serially at 37-90s per pass, and most libs need a production seam added before they are testable at all.", - "Documented that a stricter 'the suite must mention the lib' variant was written and measured first, and rejected because it newly blocked 68 libs in directories that already have suites, which seed_allowlist cannot grandfather since it refuses to grow." - ], - "verification": [ - { - "command": "bash meta/scripts/check_shell_coverage.sh --all", - "result": "pass", - "evidence": "0.31s, '37 uncovered shell librar(ies), 84 exempt', 3 BLOCK entries -- all three are meta/scripts/lib coverage-tooling libs that blocked before this change too. Under the 100% bar the same command took over 600s." - }, - { - "command": "check_shell_coverage.sh on a new lib in a dir WITH a suite, and in a dir WITHOUT one", - "result": "pass", - "evidence": "exit 0 and exit 1 respectively; the ratchet still refuses a lib whose directory has no suite. Both probe files removed." - }, - { - "command": "check_shell_coverage.sh --seed", - "result": "pass", - "evidence": "Refused, as designed: seeding would ADD the 3 tooling libs and the allowlist only ever shrinks. Allowlist left byte-identical at 84 entries." - }, - { - "command": "shellcheck and bash -n on the changed file", - "result": "pass", - "evidence": "clean; 49 lines, within the 250-line cap" - } - ], - "risks": [ - "An empty run_all.sh that sources nothing would now satisfy the gate. Accepted deliberately and recorded in the file; closing it requires a grandfathering mechanism the ratchet does not have.", - "Shell libraries will drift below whatever coverage they have today. Mitigated in that the suites already written are still run by ci_mirror.sh and the shell-tests workflow on every push." - ], - "rollback": [ - "git revert this commit to restore COVERAGE_BAR=100 and the jail-measured predicate.", - "After rollback, re-run check_shell_coverage.sh --all and expect it to take minutes again and to fail the ~11 non-exempt libs in suite-having directories." - ] -} diff --git a/docs/superpowers/evidence/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json b/docs/superpowers/evidence/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json new file mode 100644 index 00000000..edfd5a1a --- /dev/null +++ b/docs/superpowers/evidence/wsg-grabber-cloud-keep-trash-cap-2026-09-18.json @@ -0,0 +1,46 @@ +{ + "intent": "Kept /wsg/ videos go to the dufs cloud folder (or ~/Downloads as fallback) instead of a buried data dir, and trash/ is capped at the newest 200 passes with everything older deleted.", + "scope": [ + "python_pkg/wsg_grabber/paths.py: cloud_root, downloads_dir, cloud_keep_dir, keep_dir (cloud or Downloads), legacy_keep_dir, kept_candidates", + "python_pkg/wsg_grabber/_prune.py, _migrate_keep.py, housekeeping.py (new); files.remove; FileState.PURGED; store_verdicts recent_passed_names / purge_older_passes / rename_kept", + "app.py wires on_startup / after_verdict / locate_reviewed; cli.py and README drop the 'nothing is ever deleted' promise", + "Non-goal: capping kept files, touching dufs-cloud or the media sync timer" + ], + "changes": [ + "keep_dir() resolves per call to ~/data/cloud/Media//wsg when the cloud folder exists, else ~/Downloads", + "prune_trash deletes every trash/ file outside the 200 newest reviewed_at passes (orphans included) and marks the rows purged; runs at startup and after each pass", + "migrate_keep moves the legacy keep/ into the current keep dir once (dedupe by name+size, suffix + index rename on clash) and removes the empty dir", + "undo locates a kept file through kept_candidates so year rollover, cloud absence and the media sync sweep do not lose the verdict" + ], + "verification": [ + { + "command": "~/.claude/scripts/capped.sh python -m pytest python_pkg/wsg_grabber/tests -q", + "result": "pass", + "evidence": "303 passed; every python_pkg/wsg_grabber module at 100% line+branch coverage" + }, + { + "command": "HOME=/tmp/wsgfake/home XDG_DATA_HOME=$HOME/.local/share python -m python_pkg.wsg_grabber scrape --seconds 1 (copy of the real index, 23218 stand-in trash files, 50 stand-in keep files, fake data/cloud)", + "result": "pass", + "evidence": "keep.migrated moved=50; trash.pruned removed=23018 purged_rows=23018 retained=200; all 200 newest reviewed_at names present; keep/ removed; stats shows keep dir under data/cloud, and ~/Downloads after rm -rf data/cloud" + }, + { + "command": "python -m python_pkg.wsg_grabber scrape --seconds 1 (real home)", + "result": "pass", + "evidence": "keep/ 342 -> ~/data/cloud/Media/2026/wsg 647 (was 305), keep/ gone; trash 23218 files / 77G -> 200 files / 823M; stats: kept 648, passed 200, purged 23018" + }, + { + "command": "pre-commit run --files <18 changed files>", + "result": "pass", + "evidence": "no hook output beyond Passed/Skipped" + } + ], + "risks": [ + "Purging is irreversible; the first run deleted 77 GB of passed videos, as agreed (Q3).", + "A keep made while the cloud folder is absent lands in ~/Downloads and is swept by media-cloud-sync into Media//; undo follows it via kept_candidates but a same-name suffix applied by that sync would break the link.", + "Kept videos in Media//wsg now appear in the dufs gallery and get thumbnails." + ], + "rollback": [ + "git revert the commit; kept files stay in the cloud folder (undo will still find them there through the old keep_dir only if moved back by hand)", + "Validate with `python -m python_pkg.wsg_grabber stats` and the test suite" + ] +} diff --git a/python_pkg/wsg_grabber/README.md b/python_pkg/wsg_grabber/README.md index 0aafb84e..9e776c0f 100644 --- a/python_pkg/wsg_grabber/README.md +++ b/python_pkg/wsg_grabber/README.md @@ -15,33 +15,42 @@ PYTHONPATH=~/src/testsAndMisc python3 -m python_pkg.wsg_grabber stats ## Undo -Misclicked? `u` takes the last verdict back: the file returns from `keep/` or -`trash/` to `incoming/`, the index row goes back to _ready_, the counter is -corrected, and that video is on screen again. Whatever was showing when you hit -undo goes to the front of the queue, so nothing is skipped. +Misclicked? `u` takes the last verdict back: the file returns from the keep +folder or `trash/` to `incoming/`, the index row goes back to _ready_, the +counter is corrected, and that video is on screen again. Whatever was showing +when you hit undo goes to the front of the queue, so nothing is skipped. -It steps back through **any number of verdicts, and survives quitting** — the -trail lives in the index, not in memory, so reopening the reviewer can still -undo what you decided yesterday. The status bar shows how many are left -(`u undoes (17)`), and the button greys out when there is nothing to take back. -If you cleared `trash/` by hand in the meantime, undo says so in the log and -drops that entry rather than failing on it repeatedly. +It **survives quitting** — the trail lives in the index, not in memory, so +reopening the reviewer can still undo what you decided yesterday. The status +bar shows how many are left (`u undoes (17)`), and the button greys out when +there is nothing to take back. Keeps are undoable for as long as the file is +still in any place a keep can end up (`housekeeping.locate_reviewed` looks in +this year's cloud folder, earlier years', `~/Downloads`, the old `keep/`, and +wherever the media sync swept it); passes are undoable while they are among +the newest `TRASH_RETAIN` (200). If a file has gone anyway, undo says so in +the log and drops that entry rather than failing on it repeatedly. ## Storage ``` +~/data/cloud/Media//wsg/ you kept these (falls back to ~/Downloads + on a machine without the cloud folder) ~/.local/share/wsg_grabber/ ├── index.db sqlite, remembers every file ever seen ├── incoming/ downloaded, awaiting your verdict -├── keep/ you kept these -└── trash/ you passed on these — NOTHING here is ever removed automatically +└── trash/ you passed on these — only the newest 200 passes stay ``` -**No downloaded video is ever deleted.** A pass moves the file into `trash/`; -emptying it is your call. `unlink` appears only on things that are not videos — -a corrupt or abandoned `.part`, and mpv's control socket — and `files.apply_move` -refuses to overwrite an existing destination, so a move can never destroy a -video you already decided on. +A keep goes where you actually look: the dufs cloud folder, so it shows up in +the gallery and on the phone. The first start after this change moved the old +`keep/` contents across (`_migrate_keep.py`). + +A pass goes into `trash/`, and `_prune.py` deletes everything there except the +200 most recent passes — at startup and after every pass. Their index rows +become _purged_: still known, so the file is never downloaded again, but no +longer undoable. Before this cap existed 23k files / 77 GB had piled up. +`files.apply_move` still refuses to overwrite an existing destination, so a +move can never destroy a video you already decided on. ## Why it never shows you the same video twice @@ -142,7 +151,10 @@ XDG_DATA_HOME=/tmp/wsg-sandbox python3 -m python_pkg.wsg_grabber stats | `store_threads.py` | Which threads were visited, and their HTTP stamps. | | `store_verdicts.py` | The review trail that undo walks back. | | `downloader.py` | The background worker thread. | -| `files.py` | The only module that moves files. | +| `files.py` | The only module that moves or deletes files. | +| `housekeeping.py` | Around a verdict: keep migration, trash prune, finding a keep. | +| `_migrate_keep.py` | One-off move of the old `keep/` into the cloud folder. | +| `_prune.py` | Caps `trash/` at the newest 200 passes. | | `player.py` | mpv as a subprocess over its JSON IPC socket. | | `ui.py` | The only module that imports `tkinter`. | | `logs.py` | Structured JSONL event log. | diff --git a/python_pkg/wsg_grabber/_migrate_keep.py b/python_pkg/wsg_grabber/_migrate_keep.py new file mode 100644 index 00000000..f3c6655c --- /dev/null +++ b/python_pkg/wsg_grabber/_migrate_keep.py @@ -0,0 +1,74 @@ +"""One-off move of kept videos out of the data dir. + +Kept videos used to live in ``~/.local/share/wsg_grabber/keep``, which is +where nobody looks. They now go to the cloud folder (or ``~/Downloads``), and +this module carries the old directory's contents across on the first start +after the change. Idempotent: an empty or missing ``keep/`` is a no-op, and a +partial run just resumes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.wsg_grabber import files, logs, paths, store_verdicts, verdict +from python_pkg.wsg_grabber.models import FileMove + +if TYPE_CHECKING: + from pathlib import Path + import sqlite3 + + +def migrate_keep(conn: sqlite3.Connection) -> int: + """Move everything in the legacy ``keep/`` into today's keep directory. + + A file already present at the destination with the same size is a + duplicate of an earlier hand-copy and the legacy one is dropped. A name + clash with a different size gets the usual ``-2`` suffix, and the index is + told the new name so the verdict stays undoable. + + Args: + conn: Open connection. + + Returns: + int: Files moved. + """ + legacy = paths.legacy_keep_dir() + if not legacy.is_dir(): + return 0 + target = paths.keep_dir() + target.mkdir(parents=True, exist_ok=True) + taken = files.existing_names(target) + moved = 0 + for entry in sorted(legacy.iterdir()): + if not entry.is_file(): + continue + if _is_duplicate(entry, target / entry.name): + files.remove(entry) + logs.event("keep.migrate_duplicate", file=entry.name) + continue + destination = verdict.unique_destination(target, entry.name, taken) + if not files.apply_move(FileMove(md5="", src=entry, dst=destination)): + continue + taken.add(destination.name) + moved += 1 + if destination.name != entry.name: + store_verdicts.rename_kept(conn, entry.name, destination.name) + if not any(legacy.iterdir()): + legacy.rmdir() + if moved: + logs.event("keep.migrated", moved=moved, to=str(target)) + return moved + + +def _is_duplicate(source: Path, candidate: Path) -> bool: + """Report whether *candidate* is already a copy of *source*. + + Args: + source: Legacy file. + candidate: Same name at the destination. + + Returns: + bool: True when both exist with the same size. + """ + return candidate.is_file() and candidate.stat().st_size == source.stat().st_size diff --git a/python_pkg/wsg_grabber/_prune.py b/python_pkg/wsg_grabber/_prune.py new file mode 100644 index 00000000..b89ae547 --- /dev/null +++ b/python_pkg/wsg_grabber/_prune.py @@ -0,0 +1,56 @@ +"""Keeping ``trash/`` bounded. + +A pass used to be a move into a directory nobody ever emptied; 23k files and +77 GB later that was clearly wrong. The trail is now capped at the newest +``TRASH_RETAIN`` passes, which is still far more than anyone undoes. Runs at +startup and after every pass, so the directory never drifts above the cap. + +The survivor set comes from the index, not from mtimes: a verdict's +``reviewed_at`` is the fact, the file's timestamp is a side-effect of it. +Anything in ``trash/`` the index does not vouch for -- orphans from before the +``reviewed_name`` column existed -- is deleted along with the old passes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.wsg_grabber import files, logs, paths, store_verdicts +from python_pkg.wsg_grabber.constants import TRASH_RETAIN + +if TYPE_CHECKING: + import sqlite3 + + +def prune_trash(conn: sqlite3.Connection, retain: int = TRASH_RETAIN) -> int: + """Delete every trashed file except the *retain* most recent passes. + + Files go first, rows second: a crash in between leaves rows that still say + ``PASSED`` for files that are gone, which undo already copes with by + forgetting the verdict. The reverse order would leave undoable-looking + files the index had already written off. + + Args: + conn: Open connection. + retain: How many passes stay on disk and undoable. + + Returns: + int: Files deleted. + """ + trash = paths.trash_dir() + survivors = store_verdicts.recent_passed_names(conn, retain) + removed = 0 + if trash.is_dir(): + for entry in trash.iterdir(): + if entry.name in survivors or not entry.is_file(): + continue + removed += files.remove(entry) + purged = store_verdicts.purge_older_passes(conn, retain) + if removed or purged: + logs.event( + "trash.pruned", + removed=removed, + purged_rows=purged, + retained=len(survivors), + ) + return removed diff --git a/python_pkg/wsg_grabber/app.py b/python_pkg/wsg_grabber/app.py index b21cd2bd..35585acf 100644 --- a/python_pkg/wsg_grabber/app.py +++ b/python_pkg/wsg_grabber/app.py @@ -16,6 +16,7 @@ from python_pkg.wsg_grabber import ( db, downloader, files, + housekeeping, logs, net, paths, @@ -86,6 +87,7 @@ class Session: verdict.target_state(choice), move.dst.name, ) + housekeeping.after_verdict(self.conn, choice) return review.on_verdict(state, choice) def undo(self, state: SessionState) -> SessionState: @@ -100,9 +102,7 @@ class Session: action = store_verdicts.newest_verdict(self.conn, paths.incoming_dir()) if action is None: return state - source = ( - paths.keep_dir() if action.choice is Verdict.KEEP else paths.trash_dir() - ) / action.reviewed_name + source = housekeeping.locate_reviewed(action) # Mirror what commit does: never assume the old name is free again. destination = verdict.unique_destination( paths.incoming_dir(), @@ -197,6 +197,7 @@ def open_session(*, include_archive: bool = True) -> Session: paths.ensure_dirs() conn = db.open_index(paths.db_path()) store.reset_in_flight(conn) + housekeeping.on_startup(conn) events: queue.SimpleQueue[DownloadEvent] = queue.SimpleQueue() stop = threading.Event() session = Session(conn=conn, events=events, stop=stop) diff --git a/python_pkg/wsg_grabber/cli.py b/python_pkg/wsg_grabber/cli.py index 39da72f7..2799aacf 100644 --- a/python_pkg/wsg_grabber/cli.py +++ b/python_pkg/wsg_grabber/cli.py @@ -23,12 +23,13 @@ from python_pkg.wsg_grabber import ( store_threads, ui, ) +from python_pkg.wsg_grabber._logs_report import show_logs, summarise # Re-exported so `cli.show_logs` / `cli.summarise` keep working for callers and # tests; the implementation lives in _logs_report to keep this module under the # 250-line cap. __all__ is what marks them as re-exports -- the `X as X` alias # form is rejected here (ruff PLC0414) and then stripped as unused. -from python_pkg.wsg_grabber._logs_report import show_logs, summarise +from python_pkg.wsg_grabber.constants import TRASH_RETAIN from python_pkg.wsg_grabber.states import FileState if TYPE_CHECKING: @@ -49,8 +50,8 @@ _REPORT_EVERY_S = 15.0 _DESCRIPTION = ( "Scrape 4chan /wsg/ for videos and triage them with a keep/pass reviewer. " - "Nothing is ever deleted: a pass moves the file into the trash directory " - "for you to clear by hand." + "A keep goes to the cloud folder (or ~/Downloads); a pass goes to trash, " + "where only the newest passes are kept." ) @@ -159,7 +160,7 @@ def stats() -> int: _emit(f" {state.value:<12} {counts.get(state.value, 0)}") _emit(f"threads {len(store_threads.known_threads(conn))}") _emit(f"keep dir {paths.keep_dir()}") - _emit(f"trash dir {paths.trash_dir()} (never emptied automatically)") + _emit(f"trash dir {paths.trash_dir()} (newest {TRASH_RETAIN} passes kept)") finally: conn.close() return 0 diff --git a/python_pkg/wsg_grabber/constants.py b/python_pkg/wsg_grabber/constants.py index 287a9b7b..519d43d9 100644 --- a/python_pkg/wsg_grabber/constants.py +++ b/python_pkg/wsg_grabber/constants.py @@ -98,6 +98,11 @@ CONTROL_BAR_HEIGHT_PX: Final[int] = 96 WINDOW_WIDTH_PX: Final[int] = 1100 WINDOW_HEIGHT_PX: Final[int] = 800 +# How many passed videos stay in trash/ (newest first) and so stay undoable. +# Everything older is deleted for good; 23k files / 77 GB had piled up before +# this bound existed. +TRASH_RETAIN: Final[int] = 200 + KEEP_KEYS: Final[tuple[str, ...]] = ("", "") PASS_KEYS: Final[tuple[str, ...]] = ("", "", "") UNDO_KEYS: Final[tuple[str, ...]] = ("", "") diff --git a/python_pkg/wsg_grabber/files.py b/python_pkg/wsg_grabber/files.py index 720488b6..7541252a 100644 --- a/python_pkg/wsg_grabber/files.py +++ b/python_pkg/wsg_grabber/files.py @@ -1,9 +1,9 @@ """The only code in this package that moves bytes around on disk. Deliberately tiny, because everything interesting about a verdict was already -decided in :mod:`python_pkg.wsg_grabber.verdict`. There is no ``unlink`` here: -a passed video is relocated to ``trash/`` and stays there until the user -removes it. +decided in :mod:`python_pkg.wsg_grabber.verdict`. The one ``unlink`` lives +here too, so that "what touches the disk" stays a one-module question; only +:mod:`python_pkg.wsg_grabber._prune` and the keep migration call it. """ from __future__ import annotations @@ -62,3 +62,19 @@ def apply_move(move: FileMove) -> bool: move.dst.parent.mkdir(parents=True, exist_ok=True) move.src.replace(move.dst) return True + + +def remove(path: Path) -> bool: + """Delete a file for good. + + Args: + path: File to delete. + + Returns: + bool: True when a file was removed; False when nothing was there. + """ + try: + path.unlink() + except FileNotFoundError: + return False + return True diff --git a/python_pkg/wsg_grabber/housekeeping.py b/python_pkg/wsg_grabber/housekeeping.py new file mode 100644 index 00000000..d820d18a --- /dev/null +++ b/python_pkg/wsg_grabber/housekeeping.py @@ -0,0 +1,66 @@ +"""What the session does around a verdict besides the verdict itself. + +Kept out of :mod:`python_pkg.wsg_grabber.app` so that module stays about +composition and shutdown: migrating the old ``keep/`` directory, pruning +``trash/``, and working out where a kept file actually is when undo asks. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.wsg_grabber import paths +from python_pkg.wsg_grabber._migrate_keep import migrate_keep +from python_pkg.wsg_grabber._prune import prune_trash +from python_pkg.wsg_grabber.models import Verdict + +if TYPE_CHECKING: + from pathlib import Path + import sqlite3 + + from python_pkg.wsg_grabber.models import ReviewedItem + + +def on_startup(conn: sqlite3.Connection) -> None: + """Bring the on-disk layout in line with the index before reviewing. + + Args: + conn: Open connection. + """ + migrate_keep(conn) + prune_trash(conn) + + +def after_verdict(conn: sqlite3.Connection, choice: Verdict) -> None: + """Keep ``trash/`` at its cap once a pass has landed there. + + Args: + conn: Open connection. + choice: The verdict just recorded. + """ + if choice is not Verdict.KEEP: + prune_trash(conn) + + +def locate_reviewed(action: ReviewedItem) -> Path: + """Return where the file a verdict moved is sitting now. + + A pass has exactly one home. A keep may be in this year's cloud folder, + last year's, ``~/Downloads``, wherever the media sync swept it, or the + legacy ``keep/`` -- the first candidate that holds the name wins, and when + none does the current keep dir is returned so the caller's move fails + visibly rather than silently. + + Args: + action: The verdict being undone. + + Returns: + Path: Expected location of the file. + """ + if action.choice is not Verdict.KEEP: + return paths.trash_dir() / action.reviewed_name + for directory in paths.kept_candidates(): + candidate = directory / action.reviewed_name + if candidate.is_file(): + return candidate + return paths.keep_dir() / action.reviewed_name diff --git a/python_pkg/wsg_grabber/paths.py b/python_pkg/wsg_grabber/paths.py index e0f60e30..7a5af1ac 100644 --- a/python_pkg/wsg_grabber/paths.py +++ b/python_pkg/wsg_grabber/paths.py @@ -5,12 +5,19 @@ evaluated at import time, which makes it impossible for a test to redirect -- ``brother_printer.consumables`` learned that the hard way. Resolving on each call keeps the whole package redirectable from one fixture. -Nothing in this module deletes anything; ``trash_dir`` is a destination, not a -bin that gets emptied. +Kept videos leave the data dir entirely: they go to the dufs cloud folder +(``~/data/cloud/Media//wsg``) so they show up in the gallery and on the +phone, or to ``~/Downloads`` on a machine without the cloud folder. Both are +resolved per call, so the answer follows the filesystem rather than the process +start time. + +Nothing in this module deletes anything; ``trash_dir`` is a destination. +:mod:`python_pkg.wsg_grabber._prune` is what keeps it bounded. """ from __future__ import annotations +from datetime import UTC, datetime import os from pathlib import Path @@ -50,19 +57,87 @@ def incoming_dir() -> Path: return data_dir() / "incoming" +def cloud_root() -> Path: + """Return the folder dufs serves as the personal cloud. + + Returns: + Path: ``~/data/cloud`` (may not exist on this machine). + """ + return Path.home() / "data" / "cloud" + + +def downloads_dir() -> Path: + """Return the fallback destination for kept videos. + + Returns: + Path: ``~/Downloads``. + """ + return Path.home() / "Downloads" + + +def cloud_keep_dir(year: int) -> Path: + """Return the cloud folder kept videos go to for *year*. + + Args: + year: Calendar year of the keep verdict. + + Returns: + Path: ``/Media//wsg``. + """ + return cloud_root() / "Media" / str(year) / "wsg" + + def keep_dir() -> Path: """Return the directory videos are moved to when kept. + The cloud folder wins whenever it exists; otherwise ``~/Downloads``, which + is at least somewhere a person looks. + + Returns: + Path: ``/Media//wsg`` or ``~/Downloads``. + """ + if cloud_root().is_dir(): + return cloud_keep_dir(datetime.now(tz=UTC).astimezone().year) + return downloads_dir() + + +def legacy_keep_dir() -> Path: + """Return where kept videos used to go before they moved to the cloud. + + Only :mod:`python_pkg.wsg_grabber._migrate_keep` should care. + Returns: Path: ``/keep``. """ return data_dir() / "keep" +def kept_candidates() -> list[Path]: + """Return every directory a kept video may be sitting in, most likely first. + + Undo has to find a file that ``keep_dir()`` would not name today: a keep + from last year, one made while the cloud folder was absent, one the + media-cloud-sync timer has since swept out of ``~/Downloads`` into + ``Media//``, or one still in the pre-cloud ``keep/``. + + Returns: + list[Path]: Candidate directories; entries need not exist. + """ + media = cloud_root() / "Media" + return [ + keep_dir(), + *sorted(media.glob("*/wsg")), + legacy_keep_dir(), + downloads_dir(), + *sorted(p for p in media.glob("*/*") if p.is_dir() and p.name != "wsg"), + ] + + def trash_dir() -> Path: """Return the directory videos are moved to when passed. - Nothing ever removes files from here; clearing it is the user's call. + Bounded to the newest ``TRASH_RETAIN`` passes by + :mod:`python_pkg.wsg_grabber._prune`; older ones are deleted for good. Returns: Path: ``/trash``. diff --git a/python_pkg/wsg_grabber/states.py b/python_pkg/wsg_grabber/states.py index a360db6a..5ecfbf40 100644 --- a/python_pkg/wsg_grabber/states.py +++ b/python_pkg/wsg_grabber/states.py @@ -28,6 +28,9 @@ class FileState(StrEnum): KEPT = "kept" PASSED = "passed" GONE = "gone" + # A pass whose file has since been pruned from trash/: still known, so it + # is never downloaded again, but no longer undoable. + PURGED = "purged" class FileEvent(StrEnum): @@ -45,7 +48,7 @@ class FileEvent(StrEnum): TERMINAL: Final[frozenset[FileState]] = frozenset( - {FileState.KEPT, FileState.PASSED, FileState.GONE}, + {FileState.KEPT, FileState.PASSED, FileState.GONE, FileState.PURGED}, ) CLAIMABLE: Final[frozenset[FileState]] = frozenset( diff --git a/python_pkg/wsg_grabber/store_verdicts.py b/python_pkg/wsg_grabber/store_verdicts.py index 973426a9..3fd5161c 100644 --- a/python_pkg/wsg_grabber/store_verdicts.py +++ b/python_pkg/wsg_grabber/store_verdicts.py @@ -165,3 +165,72 @@ def restore_for_review(conn: sqlite3.Connection, md5_b64: str, name: str) -> Non """, (FileState.READY.value, name, md5_b64), ) + + +def recent_passed_names(conn: sqlite3.Connection, limit: int) -> set[str]: + """Return the trash names of the *limit* most recent passes. + + These are the files the trash prune keeps; everything else in ``trash/`` + goes. + + Args: + conn: Open connection. + limit: How many passes to keep. + + Returns: + set[str]: Filenames inside ``trash/``. + """ + rows = conn.execute( + """ + SELECT reviewed_name FROM files + WHERE state = ? AND reviewed_name IS NOT NULL + ORDER BY reviewed_at DESC, md5 DESC + LIMIT ? + """, + (FileState.PASSED.value, limit), + ).fetchall() + return {str(row["reviewed_name"]) for row in rows} + + +def purge_older_passes(conn: sqlite3.Connection, limit: int) -> int: + """Mark every pass beyond the *limit* newest as purged. + + A purged row keeps its md5, so the catalog never re-downloads the file, + but drops out of the undo trail because its bytes are gone. + + Args: + conn: Open connection. + limit: How many passes stay undoable. + + Returns: + int: Rows changed. + """ + cursor = conn.execute( + """ + UPDATE files + SET state = ?, reviewed_name = NULL + WHERE state = ? + AND md5 NOT IN ( + SELECT md5 FROM files + WHERE state = ? AND reviewed_name IS NOT NULL + ORDER BY reviewed_at DESC, md5 DESC + LIMIT ? + ) + """, + (FileState.PURGED.value, FileState.PASSED.value, FileState.PASSED.value, limit), + ) + return int(cursor.rowcount) + + +def rename_kept(conn: sqlite3.Connection, old: str, new: str) -> None: + """Record that a kept file changed name while being migrated. + + Args: + conn: Open connection. + old: Name the verdict recorded. + new: Name the file has now. + """ + conn.execute( + "UPDATE files SET reviewed_name = ? WHERE state = ? AND reviewed_name = ?", + (new, FileState.KEPT.value, old), + ) diff --git a/python_pkg/wsg_grabber/tests/test_app_undo.py b/python_pkg/wsg_grabber/tests/test_app_undo.py index ce11da84..0701d584 100644 --- a/python_pkg/wsg_grabber/tests/test_app_undo.py +++ b/python_pkg/wsg_grabber/tests/test_app_undo.py @@ -94,13 +94,18 @@ def test_undo_puts_a_passed_file_back(session: app.Session) -> None: def test_undo_survives_a_collision_rename(session: app.Session) -> None: - """A verdict may have renamed the file; undo must follow the real name.""" + """A verdict may have renamed the file; undo must follow the real name. + + The stray ``clip.webm`` has no index row, so the prune that follows the + pass deletes it -- which is exactly why the rename must never be undone + by guessing the original name. + """ _downloaded(session) (paths.trash_dir() / "clip.webm").write_bytes(b"older") state = session.commit(app.initial_state(session), Verdict.SKIP) assert (paths.trash_dir() / "clip-2.webm").exists() + assert not (paths.trash_dir() / "clip.webm").exists() state = session.undo(state) assert (paths.incoming_dir() / "clip.webm").read_bytes() == b"video" - assert (paths.trash_dir() / "clip.webm").read_bytes() == b"older" assert not (paths.trash_dir() / "clip-2.webm").exists() diff --git a/python_pkg/wsg_grabber/tests/test_cli.py b/python_pkg/wsg_grabber/tests/test_cli.py index d42660f8..052610b1 100644 --- a/python_pkg/wsg_grabber/tests/test_cli.py +++ b/python_pkg/wsg_grabber/tests/test_cli.py @@ -45,9 +45,10 @@ def test_parser_exposes_the_subcommands() -> None: assert parsed.seconds == pytest.approx(5.0) -def test_help_mentions_that_nothing_is_deleted() -> None: +def test_help_says_where_keeps_and_passes_go() -> None: text = cli.build_parser().format_help() - assert "Nothing is ever deleted" in text + assert "cloud folder" in text + assert "newest passes are kept" in text assert "trash" in text @@ -76,7 +77,7 @@ def test_stats_reports_every_state(capsys: pytest.CaptureFixture[str]) -> None: out = capsys.readouterr().out assert "known files 1" in out assert "new 1" in out - assert "never emptied automatically" in out + assert "newest 200 passes kept" in out def test_scrape_runs_and_shuts_down( diff --git a/python_pkg/wsg_grabber/tests/test_housekeeping.py b/python_pkg/wsg_grabber/tests/test_housekeeping.py new file mode 100644 index 00000000..b4539c4f --- /dev/null +++ b/python_pkg/wsg_grabber/tests/test_housekeeping.py @@ -0,0 +1,92 @@ +"""Tests for the around-a-verdict chores and for undo finding moved keeps.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.wsg_grabber import housekeeping, paths +from python_pkg.wsg_grabber.models import Verdict +from python_pkg.wsg_grabber.tests.conftest import item, reviewed + +if TYPE_CHECKING: + from collections.abc import Callable + + +@pytest.fixture +def prune_calls(monkeypatch: pytest.MonkeyPatch) -> list[object]: + """Replace the prune with a recorder. + + Args: + monkeypatch: Patching helper. + + Returns: + list[object]: One entry per prune call. + """ + calls: list[object] = [] + monkeypatch.setattr( + housekeeping, + "prune_trash", + calls.append, + ) + return calls + + +def test_startup_migrates_then_prunes( + monkeypatch: pytest.MonkeyPatch, + prune_calls: list[object], +) -> None: + order: list[str] = [] + monkeypatch.setattr( + housekeeping, + "migrate_keep", + lambda _conn: order.append("migrate"), + ) + housekeeping.on_startup("conn") + assert order == ["migrate"] + assert prune_calls == ["conn"] + + +@pytest.mark.parametrize( + ("choice", "expected"), + [(Verdict.SKIP, ["conn"]), (Verdict.KEEP, [])], +) +def test_only_a_pass_triggers_a_prune( + prune_calls: list[object], + choice: Verdict, + expected: list[object], +) -> None: + housekeeping.after_verdict("conn", choice) + assert prune_calls == expected + + +def test_a_pass_is_located_in_trash() -> None: + action = reviewed(item(), Verdict.SKIP) + assert housekeeping.locate_reviewed(action) == paths.trash_dir() / "a.webm" + + +@pytest.mark.parametrize( + "place", + [ + paths.keep_dir, + lambda: paths.cloud_keep_dir(2001), + paths.legacy_keep_dir, + paths.downloads_dir, + lambda: paths.cloud_root() / "Media" / "2001" / "03", + ], +) +def test_a_keep_is_found_wherever_it_ended_up(place: Callable[[], Path]) -> None: + paths.cloud_root().mkdir(parents=True) + directory = place() + directory.mkdir(parents=True, exist_ok=True) + (directory / "a.webm").write_bytes(b"v") + + action = reviewed(item(), Verdict.KEEP) + assert housekeeping.locate_reviewed(action) == directory / "a.webm" + + +def test_a_keep_that_is_nowhere_points_at_the_current_keep_dir() -> None: + action = reviewed(item(), Verdict.KEEP) + assert housekeeping.locate_reviewed(action) == paths.keep_dir() / "a.webm" diff --git a/python_pkg/wsg_grabber/tests/test_migrate_keep.py b/python_pkg/wsg_grabber/tests/test_migrate_keep.py new file mode 100644 index 00000000..0ac51258 --- /dev/null +++ b/python_pkg/wsg_grabber/tests/test_migrate_keep.py @@ -0,0 +1,130 @@ +"""Tests for moving kept videos out of the data dir.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.wsg_grabber import db, paths, store, store_verdicts +from python_pkg.wsg_grabber._migrate_keep import migrate_keep +from python_pkg.wsg_grabber.models import RemoteFile +from python_pkg.wsg_grabber.states import FileState + +if TYPE_CHECKING: + from collections.abc import Iterator + import sqlite3 + + +@pytest.fixture +def conn() -> Iterator[sqlite3.Connection]: + """Open a sandboxed index. + + Yields: + sqlite3.Connection: Closed afterwards. + """ + paths.ensure_dirs() + opened = db.open_index(paths.db_path()) + try: + yield opened + finally: + opened.close() + + +def _kept(conn: sqlite3.Connection, name: str, payload: bytes) -> None: + """Record a keep whose file sits in the legacy keep/. + + Args: + conn: Open connection. + name: Filename. + payload: File contents. + """ + store.record_files( + conn, + [ + RemoteFile( + md5=f"md5-{name}", + tim=1, + ext=".webm", + orig_name=name, + fsize=len(payload), + width=1, + height=1, + thread_no=1, + post_no=1, + ), + ], + ) + store.mark_downloaded(conn, f"md5-{name}", name) + store_verdicts.record_verdict(conn, f"md5-{name}", FileState.KEPT, name) + paths.legacy_keep_dir().mkdir(parents=True, exist_ok=True) + (paths.legacy_keep_dir() / name).write_bytes(payload) + + +def test_moves_everything_and_removes_the_empty_legacy_dir( + conn: sqlite3.Connection, +) -> None: + _kept(conn, "a.webm", b"aa") + _kept(conn, "b.webm", b"bb") + + assert migrate_keep(conn) == 2 + + assert (paths.keep_dir() / "a.webm").read_bytes() == b"aa" + assert (paths.keep_dir() / "b.webm").read_bytes() == b"bb" + assert not paths.legacy_keep_dir().exists() + + +def test_no_legacy_dir_is_a_no_op(conn: sqlite3.Connection) -> None: + assert not paths.legacy_keep_dir().exists() + assert migrate_keep(conn) == 0 + + +def test_identical_copy_at_destination_drops_the_legacy_file( + conn: sqlite3.Connection, +) -> None: + _kept(conn, "a.webm", b"aa") + paths.keep_dir().mkdir(parents=True, exist_ok=True) + (paths.keep_dir() / "a.webm").write_bytes(b"zz") + + assert migrate_keep(conn) == 0 + + assert (paths.keep_dir() / "a.webm").read_bytes() == b"zz" + assert not paths.legacy_keep_dir().exists() + + +def test_name_clash_with_different_size_is_suffixed_and_reindexed( + conn: sqlite3.Connection, +) -> None: + _kept(conn, "a.webm", b"aa") + paths.keep_dir().mkdir(parents=True, exist_ok=True) + (paths.keep_dir() / "a.webm").write_bytes(b"other") + + assert migrate_keep(conn) == 1 + + assert (paths.keep_dir() / "a.webm").read_bytes() == b"other" + assert (paths.keep_dir() / "a-2.webm").read_bytes() == b"aa" + row = conn.execute( + "SELECT reviewed_name FROM files WHERE md5 = ?", + ("md5-a.webm",), + ).fetchone() + assert row["reviewed_name"] == "a-2.webm" + + +def test_subdirectories_are_left_alone(conn: sqlite3.Connection) -> None: + paths.legacy_keep_dir().mkdir(parents=True) + (paths.legacy_keep_dir() / "nested").mkdir() + + assert migrate_keep(conn) == 0 + assert (paths.legacy_keep_dir() / "nested").is_dir() + + +def test_a_file_that_vanishes_mid_move_is_skipped( + conn: sqlite3.Connection, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _kept(conn, "a.webm", b"aa") + monkeypatch.setattr( + "python_pkg.wsg_grabber._migrate_keep.files.apply_move", + lambda _move: False, + ) + assert migrate_keep(conn) == 0 diff --git a/python_pkg/wsg_grabber/tests/test_paths.py b/python_pkg/wsg_grabber/tests/test_paths.py index e98909fd..0e24259d 100644 --- a/python_pkg/wsg_grabber/tests/test_paths.py +++ b/python_pkg/wsg_grabber/tests/test_paths.py @@ -35,13 +35,42 @@ def test_derived_paths_live_under_the_data_dir() -> None: for candidate in ( paths.db_path(), paths.incoming_dir(), - paths.keep_dir(), + paths.legacy_keep_dir(), paths.trash_dir(), paths.ipc_socket_path(), ): assert candidate.parent == root +def test_keep_dir_falls_back_to_downloads_without_the_cloud_folder() -> None: + assert not paths.cloud_root().exists() + assert paths.keep_dir() == Path.home() / "Downloads" + + +def test_keep_dir_uses_the_cloud_media_folder_for_this_year() -> None: + paths.cloud_root().mkdir(parents=True) + kept = paths.keep_dir() + assert kept.parent.parent == paths.cloud_root() / "Media" + assert kept.name == "wsg" + assert kept.parent.name.isdigit() + assert kept == paths.cloud_keep_dir(int(kept.parent.name)) + + +def test_kept_candidates_cover_every_place_a_keep_can_end_up() -> None: + media = paths.cloud_root() / "Media" + for sub in ("2025/wsg", "2026/wsg", "2026/09", "2026/notes.txt"): + (media / sub).parent.mkdir(parents=True, exist_ok=True) + (media / "2025/wsg").mkdir(parents=True) + (media / "2026/wsg").mkdir(parents=True) + (media / "2026/09").mkdir(parents=True) + (media / "2026/notes.txt").write_text("") + candidates = paths.kept_candidates() + assert candidates[0] == paths.keep_dir() + assert candidates[1:3] == [media / "2025/wsg", media / "2026/wsg"] + assert candidates[3:5] == [paths.legacy_keep_dir(), paths.downloads_dir()] + assert candidates[5:] == [media / "2026/09"] + + def test_socket_path_names_the_board() -> None: assert paths.ipc_socket_path().name == "mpv-wsg.sock" diff --git a/python_pkg/wsg_grabber/tests/test_prune.py b/python_pkg/wsg_grabber/tests/test_prune.py new file mode 100644 index 00000000..e85d0e37 --- /dev/null +++ b/python_pkg/wsg_grabber/tests/test_prune.py @@ -0,0 +1,127 @@ +"""Tests for the trash cap.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.wsg_grabber import db, files, paths, store, store_verdicts +from python_pkg.wsg_grabber._prune import prune_trash +from python_pkg.wsg_grabber.models import RemoteFile +from python_pkg.wsg_grabber.states import FileState + +if TYPE_CHECKING: + from collections.abc import Iterator + import sqlite3 + + +@pytest.fixture +def conn() -> Iterator[sqlite3.Connection]: + """Open a sandboxed index with the on-disk layout in place. + + Yields: + sqlite3.Connection: Closed afterwards. + """ + paths.ensure_dirs() + opened = db.open_index(paths.db_path()) + try: + yield opened + finally: + opened.close() + + +def _passed(conn: sqlite3.Connection, index: int, reviewed_at: str) -> str: + """Record one pass whose file sits in trash/. + + Args: + conn: Open connection. + index: Distinguishes files within a test. + reviewed_at: Verdict timestamp, so ordering is explicit. + + Returns: + str: Name of the trashed file. + """ + md5 = f"md5-{index:04d}" + name = f"{index:04d}.webm" + store.record_files( + conn, + [ + RemoteFile( + md5=md5, + tim=index, + ext=".webm", + orig_name=name, + fsize=1, + width=1, + height=1, + thread_no=1, + post_no=index, + ), + ], + ) + store.mark_downloaded(conn, md5, name) + store_verdicts.record_verdict(conn, md5, FileState.PASSED, name) + conn.execute("UPDATE files SET reviewed_at = ? WHERE md5 = ?", (reviewed_at, md5)) + (paths.trash_dir() / name).write_bytes(b"v") + return name + + +def test_keeps_only_the_newest_passes(conn: sqlite3.Connection) -> None: + names = [_passed(conn, i, f"2026-09-{i + 1:02d}") for i in range(5)] + + assert prune_trash(conn, retain=2) == 3 + + assert sorted(p.name for p in paths.trash_dir().iterdir()) == names[3:] + assert store.state_of(conn, "md5-0000") is FileState.PURGED + assert store.state_of(conn, "md5-0004") is FileState.PASSED + assert store_verdicts.reviewed_count(conn) == 2 + + +def test_orphans_without_an_index_row_are_deleted_too( + conn: sqlite3.Connection, +) -> None: + _passed(conn, 1, "2026-09-01") + (paths.trash_dir() / "orphan.webm").write_bytes(b"?") + (paths.trash_dir() / "subdir").mkdir() + + assert prune_trash(conn, retain=5) == 1 + + assert not (paths.trash_dir() / "orphan.webm").exists() + assert (paths.trash_dir() / "0001.webm").exists() + assert (paths.trash_dir() / "subdir").is_dir() + + +def test_under_the_cap_is_a_no_op(conn: sqlite3.Connection) -> None: + _passed(conn, 1, "2026-09-01") + assert prune_trash(conn, retain=5) == 0 + assert store.state_of(conn, "md5-0001") is FileState.PASSED + + +def test_missing_trash_dir_is_tolerated(conn: sqlite3.Connection) -> None: + paths.trash_dir().rmdir() + assert prune_trash(conn) == 0 + + +def test_a_pass_whose_file_was_forgotten_is_purged(conn: sqlite3.Connection) -> None: + """A row with no reviewed_name cannot be undone, so it is written off.""" + _passed(conn, 1, "2026-09-01") + store_verdicts.forget_verdict(conn, "md5-0001") + + prune_trash(conn, retain=5) + + assert store.state_of(conn, "md5-0001") is FileState.PURGED + + +def test_purged_rows_are_still_known_to_the_catalog(conn: sqlite3.Connection) -> None: + _passed(conn, 1, "2026-09-01") + prune_trash(conn, retain=0) + assert "md5-0001" in store.known_md5s(conn) + + +def test_remove_reports_whether_anything_was_there() -> None: + target = paths.data_dir() / "x" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"") + assert files.remove(target) is True + assert files.remove(target) is False diff --git a/python_pkg/wsg_grabber/tests/test_states.py b/python_pkg/wsg_grabber/tests/test_states.py index c48a6e21..bd6c2268 100644 --- a/python_pkg/wsg_grabber/tests/test_states.py +++ b/python_pkg/wsg_grabber/tests/test_states.py @@ -80,8 +80,13 @@ def test_illegal_transition_from_a_terminal_state_raises() -> None: next_state(FileState.GONE, FileEvent.CLAIMED, 0) -def test_terminal_states_are_exactly_the_three_end_points() -> None: - assert {FileState.KEPT, FileState.PASSED, FileState.GONE} == TERMINAL +def test_terminal_states_are_exactly_the_four_end_points() -> None: + assert { + FileState.KEPT, + FileState.PASSED, + FileState.GONE, + FileState.PURGED, + } == TERMINAL for state in TERMINAL: assert is_terminal(state) for state in CLAIMABLE: