mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 18:47:04 +01:00
wsg_grabber: keep to the cloud folder, cap trash at 200 passes
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (mypy, mypy) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (pylint, pylint) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (rest, ) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (shellcheck, shellcheck) (push) Waiting to run
Python tests / test (push) Waiting to run
Shell tests / shell-tests (push) Waiting to run
Shell tests / lib-tests (push) Waiting to run
Shell tests / arch-tests (push) Waiting to run
Some checks are pending
dependency freshness / dependencies on newest stable (push) Waiting to run
md-naming / markdown naming convention (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (mypy, mypy) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (pylint, pylint) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (rest, ) (push) Waiting to run
Pre-commit checks / pre-commit (${{ matrix.group }}) (shellcheck, shellcheck) (push) Waiting to run
Python tests / test (push) Waiting to run
Shell tests / shell-tests (push) Waiting to run
Shell tests / lib-tests (push) Waiting to run
Shell tests / arch-tests (push) Waiting to run
Kept videos land in ~/data/cloud/Media/<year>/wsg (~/Downloads without the cloud folder); trash/ keeps only the newest 200 passes and marks the rest purged. Legacy keep/ is migrated once. Undo follows a keep through every place it can end up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01F8G3BNETGmbr6aM5cc18g4
This commit is contained in:
parent
375555fa37
commit
33f0fee342
@ -0,0 +1,18 @@
|
||||
{
|
||||
"title": "wsg_grabber: keep to the cloud folder, cap trash at 200 passes",
|
||||
"objective": "Kept videos land somewhere a person looks (~/data/cloud/Media/<year>/wsg, ~/Downloads without the cloud folder) instead of ~/.local/share/wsg_grabber/keep, and trash/ stops growing without bound: only the 200 most recent passes stay on disk and undoable, everything older is deleted for good.",
|
||||
"acceptance_criteria": [
|
||||
"`wsg-grabber stats` prints `keep dir ~/data/cloud/Media/<this year>/wsg`, and a keep verdict moves the file there.",
|
||||
"With HOME pointed at a tree without data/cloud, a keep lands in $HOME/Downloads.",
|
||||
"After startup and after every pass, `ls trash | wc -l` is at most 200 and the survivors are exactly the 200 newest reviewed_at PASSED rows; orphans without an index row are deleted too.",
|
||||
"Rows for deleted trash files become the new terminal state `purged`: still in known_md5s (never re-downloaded), excluded from the undo trail.",
|
||||
"The old keep/ contents are migrated on first start (same name+size at the destination drops the duplicate, other clashes get a -2 suffix and the index row is renamed), and the empty keep/ is removed.",
|
||||
"Undo still restores a keep from wherever it ended up (this year's cloud folder, earlier years', ~/Downloads, the legacy keep/, Media/<year>/<month>) and a pass from trash/."
|
||||
],
|
||||
"out_of_scope": [
|
||||
"Capping kept files: they are the product and stay forever.",
|
||||
"A schema column for the keep directory; the candidate search covers every location without one.",
|
||||
"Any change to dufs-cloud, the gallery, or the media-cloud-sync timer."
|
||||
],
|
||||
"verifier": "pytest python_pkg/wsg_grabber/tests (303 passed, every wsg_grabber module 100% branch coverage); rehearsal in a fabricated HOME with a copy of the real index and stand-in files; then the real run: keep/ 342 -> cloud 647, trash 23218 files / 77 GB -> 200 files / 823 MB, 23018 rows purged."
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
{
|
||||
"intent": "Stop enforcing a measured 100% line-coverage bar on shell libraries, which priced the remaining work at several days, and return the ratchet to the presence check it was before 2026-08-22.",
|
||||
"scope": [
|
||||
"meta/scripts/lib/shell_coverage_bar.sh",
|
||||
"Non-goal: deleting shell_coverage_jail.sh, which is still used deliberately and by ci_mirror.sh",
|
||||
"Non-goal: changing python_pkg's 100% branch-coverage bar, which is already met"
|
||||
],
|
||||
"changes": [
|
||||
"is_covered() is once again 'the lib's directory has a tests/run_all.sh', replacing the jail-measured COVERAGE_BAR=100 introduced in bca70b67.",
|
||||
"Recorded why in the file: 22 libs in fixes/lib required 34 hand-written test files, the jail runs serially at 37-90s per pass, and most libs need a production seam added before they are testable at all.",
|
||||
"Documented that a stricter 'the suite must mention the lib' variant was written and measured first, and rejected because it newly blocked 68 libs in directories that already have suites, which seed_allowlist cannot grandfather since it refuses to grow."
|
||||
],
|
||||
"verification": [
|
||||
{
|
||||
"command": "bash meta/scripts/check_shell_coverage.sh --all",
|
||||
"result": "pass",
|
||||
"evidence": "0.31s, '37 uncovered shell librar(ies), 84 exempt', 3 BLOCK entries -- all three are meta/scripts/lib coverage-tooling libs that blocked before this change too. Under the 100% bar the same command took over 600s."
|
||||
},
|
||||
{
|
||||
"command": "check_shell_coverage.sh on a new lib in a dir WITH a suite, and in a dir WITHOUT one",
|
||||
"result": "pass",
|
||||
"evidence": "exit 0 and exit 1 respectively; the ratchet still refuses a lib whose directory has no suite. Both probe files removed."
|
||||
},
|
||||
{
|
||||
"command": "check_shell_coverage.sh --seed",
|
||||
"result": "pass",
|
||||
"evidence": "Refused, as designed: seeding would ADD the 3 tooling libs and the allowlist only ever shrinks. Allowlist left byte-identical at 84 entries."
|
||||
},
|
||||
{
|
||||
"command": "shellcheck and bash -n on the changed file",
|
||||
"result": "pass",
|
||||
"evidence": "clean; 49 lines, within the 250-line cap"
|
||||
}
|
||||
],
|
||||
"risks": [
|
||||
"An empty run_all.sh that sources nothing would now satisfy the gate. Accepted deliberately and recorded in the file; closing it requires a grandfathering mechanism the ratchet does not have.",
|
||||
"Shell libraries will drift below whatever coverage they have today. Mitigated in that the suites already written are still run by ci_mirror.sh and the shell-tests workflow on every push."
|
||||
],
|
||||
"rollback": [
|
||||
"git revert this commit to restore COVERAGE_BAR=100 and the jail-measured predicate.",
|
||||
"After rollback, re-run check_shell_coverage.sh --all and expect it to take minutes again and to fail the ~11 non-exempt libs in suite-having directories."
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
{
|
||||
"intent": "Kept /wsg/ videos go to the dufs cloud folder (or ~/Downloads as fallback) instead of a buried data dir, and trash/ is capped at the newest 200 passes with everything older deleted.",
|
||||
"scope": [
|
||||
"python_pkg/wsg_grabber/paths.py: cloud_root, downloads_dir, cloud_keep_dir, keep_dir (cloud or Downloads), legacy_keep_dir, kept_candidates",
|
||||
"python_pkg/wsg_grabber/_prune.py, _migrate_keep.py, housekeeping.py (new); files.remove; FileState.PURGED; store_verdicts recent_passed_names / purge_older_passes / rename_kept",
|
||||
"app.py wires on_startup / after_verdict / locate_reviewed; cli.py and README drop the 'nothing is ever deleted' promise",
|
||||
"Non-goal: capping kept files, touching dufs-cloud or the media sync timer"
|
||||
],
|
||||
"changes": [
|
||||
"keep_dir() resolves per call to ~/data/cloud/Media/<year>/wsg when the cloud folder exists, else ~/Downloads",
|
||||
"prune_trash deletes every trash/ file outside the 200 newest reviewed_at passes (orphans included) and marks the rows purged; runs at startup and after each pass",
|
||||
"migrate_keep moves the legacy keep/ into the current keep dir once (dedupe by name+size, suffix + index rename on clash) and removes the empty dir",
|
||||
"undo locates a kept file through kept_candidates so year rollover, cloud absence and the media sync sweep do not lose the verdict"
|
||||
],
|
||||
"verification": [
|
||||
{
|
||||
"command": "~/.claude/scripts/capped.sh python -m pytest python_pkg/wsg_grabber/tests -q",
|
||||
"result": "pass",
|
||||
"evidence": "303 passed; every python_pkg/wsg_grabber module at 100% line+branch coverage"
|
||||
},
|
||||
{
|
||||
"command": "HOME=/tmp/wsgfake/home XDG_DATA_HOME=$HOME/.local/share python -m python_pkg.wsg_grabber scrape --seconds 1 (copy of the real index, 23218 stand-in trash files, 50 stand-in keep files, fake data/cloud)",
|
||||
"result": "pass",
|
||||
"evidence": "keep.migrated moved=50; trash.pruned removed=23018 purged_rows=23018 retained=200; all 200 newest reviewed_at names present; keep/ removed; stats shows keep dir under data/cloud, and ~/Downloads after rm -rf data/cloud"
|
||||
},
|
||||
{
|
||||
"command": "python -m python_pkg.wsg_grabber scrape --seconds 1 (real home)",
|
||||
"result": "pass",
|
||||
"evidence": "keep/ 342 -> ~/data/cloud/Media/2026/wsg 647 (was 305), keep/ gone; trash 23218 files / 77G -> 200 files / 823M; stats: kept 648, passed 200, purged 23018"
|
||||
},
|
||||
{
|
||||
"command": "pre-commit run --files <18 changed files>",
|
||||
"result": "pass",
|
||||
"evidence": "no hook output beyond Passed/Skipped"
|
||||
}
|
||||
],
|
||||
"risks": [
|
||||
"Purging is irreversible; the first run deleted 77 GB of passed videos, as agreed (Q3).",
|
||||
"A keep made while the cloud folder is absent lands in ~/Downloads and is swept by media-cloud-sync into Media/<year>/<month>; undo follows it via kept_candidates but a same-name suffix applied by that sync would break the link.",
|
||||
"Kept videos in Media/<year>/wsg now appear in the dufs gallery and get thumbnails."
|
||||
],
|
||||
"rollback": [
|
||||
"git revert the commit; kept files stay in the cloud folder (undo will still find them there through the old keep_dir only if moved back by hand)",
|
||||
"Validate with `python -m python_pkg.wsg_grabber stats` and the test suite"
|
||||
]
|
||||
}
|
||||
@ -15,33 +15,42 @@ PYTHONPATH=~/src/testsAndMisc python3 -m python_pkg.wsg_grabber stats
|
||||
|
||||
## Undo
|
||||
|
||||
Misclicked? `u` takes the last verdict back: the file returns from `keep/` or
|
||||
`trash/` to `incoming/`, the index row goes back to _ready_, the counter is
|
||||
corrected, and that video is on screen again. Whatever was showing when you hit
|
||||
undo goes to the front of the queue, so nothing is skipped.
|
||||
Misclicked? `u` takes the last verdict back: the file returns from the keep
|
||||
folder or `trash/` to `incoming/`, the index row goes back to _ready_, the
|
||||
counter is corrected, and that video is on screen again. Whatever was showing
|
||||
when you hit undo goes to the front of the queue, so nothing is skipped.
|
||||
|
||||
It steps back through **any number of verdicts, and survives quitting** — the
|
||||
trail lives in the index, not in memory, so reopening the reviewer can still
|
||||
undo what you decided yesterday. The status bar shows how many are left
|
||||
(`u undoes (17)`), and the button greys out when there is nothing to take back.
|
||||
If you cleared `trash/` by hand in the meantime, undo says so in the log and
|
||||
drops that entry rather than failing on it repeatedly.
|
||||
It **survives quitting** — the trail lives in the index, not in memory, so
|
||||
reopening the reviewer can still undo what you decided yesterday. The status
|
||||
bar shows how many are left (`u undoes (17)`), and the button greys out when
|
||||
there is nothing to take back. Keeps are undoable for as long as the file is
|
||||
still in any place a keep can end up (`housekeeping.locate_reviewed` looks in
|
||||
this year's cloud folder, earlier years', `~/Downloads`, the old `keep/`, and
|
||||
wherever the media sync swept it); passes are undoable while they are among
|
||||
the newest `TRASH_RETAIN` (200). If a file has gone anyway, undo says so in
|
||||
the log and drops that entry rather than failing on it repeatedly.
|
||||
|
||||
## Storage
|
||||
|
||||
```
|
||||
~/data/cloud/Media/<year>/wsg/ you kept these (falls back to ~/Downloads
|
||||
on a machine without the cloud folder)
|
||||
~/.local/share/wsg_grabber/
|
||||
├── index.db sqlite, remembers every file ever seen
|
||||
├── incoming/ downloaded, awaiting your verdict
|
||||
├── keep/ you kept these
|
||||
└── trash/ you passed on these — NOTHING here is ever removed automatically
|
||||
└── trash/ you passed on these — only the newest 200 passes stay
|
||||
```
|
||||
|
||||
**No downloaded video is ever deleted.** A pass moves the file into `trash/`;
|
||||
emptying it is your call. `unlink` appears only on things that are not videos —
|
||||
a corrupt or abandoned `.part`, and mpv's control socket — and `files.apply_move`
|
||||
refuses to overwrite an existing destination, so a move can never destroy a
|
||||
video you already decided on.
|
||||
A keep goes where you actually look: the dufs cloud folder, so it shows up in
|
||||
the gallery and on the phone. The first start after this change moved the old
|
||||
`keep/` contents across (`_migrate_keep.py`).
|
||||
|
||||
A pass goes into `trash/`, and `_prune.py` deletes everything there except the
|
||||
200 most recent passes — at startup and after every pass. Their index rows
|
||||
become _purged_: still known, so the file is never downloaded again, but no
|
||||
longer undoable. Before this cap existed 23k files / 77 GB had piled up.
|
||||
`files.apply_move` still refuses to overwrite an existing destination, so a
|
||||
move can never destroy a video you already decided on.
|
||||
|
||||
## Why it never shows you the same video twice
|
||||
|
||||
@ -142,7 +151,10 @@ XDG_DATA_HOME=/tmp/wsg-sandbox python3 -m python_pkg.wsg_grabber stats
|
||||
| `store_threads.py` | Which threads were visited, and their HTTP stamps. |
|
||||
| `store_verdicts.py` | The review trail that undo walks back. |
|
||||
| `downloader.py` | The background worker thread. |
|
||||
| `files.py` | The only module that moves files. |
|
||||
| `files.py` | The only module that moves or deletes files. |
|
||||
| `housekeeping.py` | Around a verdict: keep migration, trash prune, finding a keep. |
|
||||
| `_migrate_keep.py` | One-off move of the old `keep/` into the cloud folder. |
|
||||
| `_prune.py` | Caps `trash/` at the newest 200 passes. |
|
||||
| `player.py` | mpv as a subprocess over its JSON IPC socket. |
|
||||
| `ui.py` | The only module that imports `tkinter`. |
|
||||
| `logs.py` | Structured JSONL event log. |
|
||||
|
||||
74
python_pkg/wsg_grabber/_migrate_keep.py
Normal file
74
python_pkg/wsg_grabber/_migrate_keep.py
Normal file
@ -0,0 +1,74 @@
|
||||
"""One-off move of kept videos out of the data dir.
|
||||
|
||||
Kept videos used to live in ``~/.local/share/wsg_grabber/keep``, which is
|
||||
where nobody looks. They now go to the cloud folder (or ``~/Downloads``), and
|
||||
this module carries the old directory's contents across on the first start
|
||||
after the change. Idempotent: an empty or missing ``keep/`` is a no-op, and a
|
||||
partial run just resumes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python_pkg.wsg_grabber import files, logs, paths, store_verdicts, verdict
|
||||
from python_pkg.wsg_grabber.models import FileMove
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
|
||||
|
||||
def migrate_keep(conn: sqlite3.Connection) -> int:
|
||||
"""Move everything in the legacy ``keep/`` into today's keep directory.
|
||||
|
||||
A file already present at the destination with the same size is a
|
||||
duplicate of an earlier hand-copy and the legacy one is dropped. A name
|
||||
clash with a different size gets the usual ``-2`` suffix, and the index is
|
||||
told the new name so the verdict stays undoable.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
int: Files moved.
|
||||
"""
|
||||
legacy = paths.legacy_keep_dir()
|
||||
if not legacy.is_dir():
|
||||
return 0
|
||||
target = paths.keep_dir()
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
taken = files.existing_names(target)
|
||||
moved = 0
|
||||
for entry in sorted(legacy.iterdir()):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
if _is_duplicate(entry, target / entry.name):
|
||||
files.remove(entry)
|
||||
logs.event("keep.migrate_duplicate", file=entry.name)
|
||||
continue
|
||||
destination = verdict.unique_destination(target, entry.name, taken)
|
||||
if not files.apply_move(FileMove(md5="", src=entry, dst=destination)):
|
||||
continue
|
||||
taken.add(destination.name)
|
||||
moved += 1
|
||||
if destination.name != entry.name:
|
||||
store_verdicts.rename_kept(conn, entry.name, destination.name)
|
||||
if not any(legacy.iterdir()):
|
||||
legacy.rmdir()
|
||||
if moved:
|
||||
logs.event("keep.migrated", moved=moved, to=str(target))
|
||||
return moved
|
||||
|
||||
|
||||
def _is_duplicate(source: Path, candidate: Path) -> bool:
|
||||
"""Report whether *candidate* is already a copy of *source*.
|
||||
|
||||
Args:
|
||||
source: Legacy file.
|
||||
candidate: Same name at the destination.
|
||||
|
||||
Returns:
|
||||
bool: True when both exist with the same size.
|
||||
"""
|
||||
return candidate.is_file() and candidate.stat().st_size == source.stat().st_size
|
||||
56
python_pkg/wsg_grabber/_prune.py
Normal file
56
python_pkg/wsg_grabber/_prune.py
Normal file
@ -0,0 +1,56 @@
|
||||
"""Keeping ``trash/`` bounded.
|
||||
|
||||
A pass used to be a move into a directory nobody ever emptied; 23k files and
|
||||
77 GB later that was clearly wrong. The trail is now capped at the newest
|
||||
``TRASH_RETAIN`` passes, which is still far more than anyone undoes. Runs at
|
||||
startup and after every pass, so the directory never drifts above the cap.
|
||||
|
||||
The survivor set comes from the index, not from mtimes: a verdict's
|
||||
``reviewed_at`` is the fact, the file's timestamp is a side-effect of it.
|
||||
Anything in ``trash/`` the index does not vouch for -- orphans from before the
|
||||
``reviewed_name`` column existed -- is deleted along with the old passes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python_pkg.wsg_grabber import files, logs, paths, store_verdicts
|
||||
from python_pkg.wsg_grabber.constants import TRASH_RETAIN
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import sqlite3
|
||||
|
||||
|
||||
def prune_trash(conn: sqlite3.Connection, retain: int = TRASH_RETAIN) -> int:
|
||||
"""Delete every trashed file except the *retain* most recent passes.
|
||||
|
||||
Files go first, rows second: a crash in between leaves rows that still say
|
||||
``PASSED`` for files that are gone, which undo already copes with by
|
||||
forgetting the verdict. The reverse order would leave undoable-looking
|
||||
files the index had already written off.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
retain: How many passes stay on disk and undoable.
|
||||
|
||||
Returns:
|
||||
int: Files deleted.
|
||||
"""
|
||||
trash = paths.trash_dir()
|
||||
survivors = store_verdicts.recent_passed_names(conn, retain)
|
||||
removed = 0
|
||||
if trash.is_dir():
|
||||
for entry in trash.iterdir():
|
||||
if entry.name in survivors or not entry.is_file():
|
||||
continue
|
||||
removed += files.remove(entry)
|
||||
purged = store_verdicts.purge_older_passes(conn, retain)
|
||||
if removed or purged:
|
||||
logs.event(
|
||||
"trash.pruned",
|
||||
removed=removed,
|
||||
purged_rows=purged,
|
||||
retained=len(survivors),
|
||||
)
|
||||
return removed
|
||||
@ -16,6 +16,7 @@ from python_pkg.wsg_grabber import (
|
||||
db,
|
||||
downloader,
|
||||
files,
|
||||
housekeeping,
|
||||
logs,
|
||||
net,
|
||||
paths,
|
||||
@ -86,6 +87,7 @@ class Session:
|
||||
verdict.target_state(choice),
|
||||
move.dst.name,
|
||||
)
|
||||
housekeeping.after_verdict(self.conn, choice)
|
||||
return review.on_verdict(state, choice)
|
||||
|
||||
def undo(self, state: SessionState) -> SessionState:
|
||||
@ -100,9 +102,7 @@ class Session:
|
||||
action = store_verdicts.newest_verdict(self.conn, paths.incoming_dir())
|
||||
if action is None:
|
||||
return state
|
||||
source = (
|
||||
paths.keep_dir() if action.choice is Verdict.KEEP else paths.trash_dir()
|
||||
) / action.reviewed_name
|
||||
source = housekeeping.locate_reviewed(action)
|
||||
# Mirror what commit does: never assume the old name is free again.
|
||||
destination = verdict.unique_destination(
|
||||
paths.incoming_dir(),
|
||||
@ -197,6 +197,7 @@ def open_session(*, include_archive: bool = True) -> Session:
|
||||
paths.ensure_dirs()
|
||||
conn = db.open_index(paths.db_path())
|
||||
store.reset_in_flight(conn)
|
||||
housekeeping.on_startup(conn)
|
||||
events: queue.SimpleQueue[DownloadEvent] = queue.SimpleQueue()
|
||||
stop = threading.Event()
|
||||
session = Session(conn=conn, events=events, stop=stop)
|
||||
|
||||
@ -23,12 +23,13 @@ from python_pkg.wsg_grabber import (
|
||||
store_threads,
|
||||
ui,
|
||||
)
|
||||
from python_pkg.wsg_grabber._logs_report import show_logs, summarise
|
||||
|
||||
# Re-exported so `cli.show_logs` / `cli.summarise` keep working for callers and
|
||||
# tests; the implementation lives in _logs_report to keep this module under the
|
||||
# 250-line cap. __all__ is what marks them as re-exports -- the `X as X` alias
|
||||
# form is rejected here (ruff PLC0414) and then stripped as unused.
|
||||
from python_pkg.wsg_grabber._logs_report import show_logs, summarise
|
||||
from python_pkg.wsg_grabber.constants import TRASH_RETAIN
|
||||
from python_pkg.wsg_grabber.states import FileState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@ -49,8 +50,8 @@ _REPORT_EVERY_S = 15.0
|
||||
|
||||
_DESCRIPTION = (
|
||||
"Scrape 4chan /wsg/ for videos and triage them with a keep/pass reviewer. "
|
||||
"Nothing is ever deleted: a pass moves the file into the trash directory "
|
||||
"for you to clear by hand."
|
||||
"A keep goes to the cloud folder (or ~/Downloads); a pass goes to trash, "
|
||||
"where only the newest passes are kept."
|
||||
)
|
||||
|
||||
|
||||
@ -159,7 +160,7 @@ def stats() -> int:
|
||||
_emit(f" {state.value:<12} {counts.get(state.value, 0)}")
|
||||
_emit(f"threads {len(store_threads.known_threads(conn))}")
|
||||
_emit(f"keep dir {paths.keep_dir()}")
|
||||
_emit(f"trash dir {paths.trash_dir()} (never emptied automatically)")
|
||||
_emit(f"trash dir {paths.trash_dir()} (newest {TRASH_RETAIN} passes kept)")
|
||||
finally:
|
||||
conn.close()
|
||||
return 0
|
||||
|
||||
@ -98,6 +98,11 @@ CONTROL_BAR_HEIGHT_PX: Final[int] = 96
|
||||
WINDOW_WIDTH_PX: Final[int] = 1100
|
||||
WINDOW_HEIGHT_PX: Final[int] = 800
|
||||
|
||||
# How many passed videos stay in trash/ (newest first) and so stay undoable.
|
||||
# Everything older is deleted for good; 23k files / 77 GB had piled up before
|
||||
# this bound existed.
|
||||
TRASH_RETAIN: Final[int] = 200
|
||||
|
||||
KEEP_KEYS: Final[tuple[str, ...]] = ("<k>", "<Right>")
|
||||
PASS_KEYS: Final[tuple[str, ...]] = ("<j>", "<space>", "<Left>")
|
||||
UNDO_KEYS: Final[tuple[str, ...]] = ("<u>", "<BackSpace>")
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
"""The only code in this package that moves bytes around on disk.
|
||||
|
||||
Deliberately tiny, because everything interesting about a verdict was already
|
||||
decided in :mod:`python_pkg.wsg_grabber.verdict`. There is no ``unlink`` here:
|
||||
a passed video is relocated to ``trash/`` and stays there until the user
|
||||
removes it.
|
||||
decided in :mod:`python_pkg.wsg_grabber.verdict`. The one ``unlink`` lives
|
||||
here too, so that "what touches the disk" stays a one-module question; only
|
||||
:mod:`python_pkg.wsg_grabber._prune` and the keep migration call it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@ -62,3 +62,19 @@ def apply_move(move: FileMove) -> bool:
|
||||
move.dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
move.src.replace(move.dst)
|
||||
return True
|
||||
|
||||
|
||||
def remove(path: Path) -> bool:
|
||||
"""Delete a file for good.
|
||||
|
||||
Args:
|
||||
path: File to delete.
|
||||
|
||||
Returns:
|
||||
bool: True when a file was removed; False when nothing was there.
|
||||
"""
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return True
|
||||
|
||||
66
python_pkg/wsg_grabber/housekeeping.py
Normal file
66
python_pkg/wsg_grabber/housekeeping.py
Normal file
@ -0,0 +1,66 @@
|
||||
"""What the session does around a verdict besides the verdict itself.
|
||||
|
||||
Kept out of :mod:`python_pkg.wsg_grabber.app` so that module stays about
|
||||
composition and shutdown: migrating the old ``keep/`` directory, pruning
|
||||
``trash/``, and working out where a kept file actually is when undo asks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python_pkg.wsg_grabber import paths
|
||||
from python_pkg.wsg_grabber._migrate_keep import migrate_keep
|
||||
from python_pkg.wsg_grabber._prune import prune_trash
|
||||
from python_pkg.wsg_grabber.models import Verdict
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
|
||||
from python_pkg.wsg_grabber.models import ReviewedItem
|
||||
|
||||
|
||||
def on_startup(conn: sqlite3.Connection) -> None:
|
||||
"""Bring the on-disk layout in line with the index before reviewing.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
"""
|
||||
migrate_keep(conn)
|
||||
prune_trash(conn)
|
||||
|
||||
|
||||
def after_verdict(conn: sqlite3.Connection, choice: Verdict) -> None:
|
||||
"""Keep ``trash/`` at its cap once a pass has landed there.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
choice: The verdict just recorded.
|
||||
"""
|
||||
if choice is not Verdict.KEEP:
|
||||
prune_trash(conn)
|
||||
|
||||
|
||||
def locate_reviewed(action: ReviewedItem) -> Path:
|
||||
"""Return where the file a verdict moved is sitting now.
|
||||
|
||||
A pass has exactly one home. A keep may be in this year's cloud folder,
|
||||
last year's, ``~/Downloads``, wherever the media sync swept it, or the
|
||||
legacy ``keep/`` -- the first candidate that holds the name wins, and when
|
||||
none does the current keep dir is returned so the caller's move fails
|
||||
visibly rather than silently.
|
||||
|
||||
Args:
|
||||
action: The verdict being undone.
|
||||
|
||||
Returns:
|
||||
Path: Expected location of the file.
|
||||
"""
|
||||
if action.choice is not Verdict.KEEP:
|
||||
return paths.trash_dir() / action.reviewed_name
|
||||
for directory in paths.kept_candidates():
|
||||
candidate = directory / action.reviewed_name
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
return paths.keep_dir() / action.reviewed_name
|
||||
@ -5,12 +5,19 @@ evaluated at import time, which makes it impossible for a test to redirect --
|
||||
``brother_printer.consumables`` learned that the hard way. Resolving on each
|
||||
call keeps the whole package redirectable from one fixture.
|
||||
|
||||
Nothing in this module deletes anything; ``trash_dir`` is a destination, not a
|
||||
bin that gets emptied.
|
||||
Kept videos leave the data dir entirely: they go to the dufs cloud folder
|
||||
(``~/data/cloud/Media/<year>/wsg``) so they show up in the gallery and on the
|
||||
phone, or to ``~/Downloads`` on a machine without the cloud folder. Both are
|
||||
resolved per call, so the answer follows the filesystem rather than the process
|
||||
start time.
|
||||
|
||||
Nothing in this module deletes anything; ``trash_dir`` is a destination.
|
||||
:mod:`python_pkg.wsg_grabber._prune` is what keeps it bounded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
@ -50,19 +57,87 @@ def incoming_dir() -> Path:
|
||||
return data_dir() / "incoming"
|
||||
|
||||
|
||||
def cloud_root() -> Path:
|
||||
"""Return the folder dufs serves as the personal cloud.
|
||||
|
||||
Returns:
|
||||
Path: ``~/data/cloud`` (may not exist on this machine).
|
||||
"""
|
||||
return Path.home() / "data" / "cloud"
|
||||
|
||||
|
||||
def downloads_dir() -> Path:
|
||||
"""Return the fallback destination for kept videos.
|
||||
|
||||
Returns:
|
||||
Path: ``~/Downloads``.
|
||||
"""
|
||||
return Path.home() / "Downloads"
|
||||
|
||||
|
||||
def cloud_keep_dir(year: int) -> Path:
|
||||
"""Return the cloud folder kept videos go to for *year*.
|
||||
|
||||
Args:
|
||||
year: Calendar year of the keep verdict.
|
||||
|
||||
Returns:
|
||||
Path: ``<cloud>/Media/<year>/wsg``.
|
||||
"""
|
||||
return cloud_root() / "Media" / str(year) / "wsg"
|
||||
|
||||
|
||||
def keep_dir() -> Path:
|
||||
"""Return the directory videos are moved to when kept.
|
||||
|
||||
The cloud folder wins whenever it exists; otherwise ``~/Downloads``, which
|
||||
is at least somewhere a person looks.
|
||||
|
||||
Returns:
|
||||
Path: ``<cloud>/Media/<this year>/wsg`` or ``~/Downloads``.
|
||||
"""
|
||||
if cloud_root().is_dir():
|
||||
return cloud_keep_dir(datetime.now(tz=UTC).astimezone().year)
|
||||
return downloads_dir()
|
||||
|
||||
|
||||
def legacy_keep_dir() -> Path:
|
||||
"""Return where kept videos used to go before they moved to the cloud.
|
||||
|
||||
Only :mod:`python_pkg.wsg_grabber._migrate_keep` should care.
|
||||
|
||||
Returns:
|
||||
Path: ``<data dir>/keep``.
|
||||
"""
|
||||
return data_dir() / "keep"
|
||||
|
||||
|
||||
def kept_candidates() -> list[Path]:
|
||||
"""Return every directory a kept video may be sitting in, most likely first.
|
||||
|
||||
Undo has to find a file that ``keep_dir()`` would not name today: a keep
|
||||
from last year, one made while the cloud folder was absent, one the
|
||||
media-cloud-sync timer has since swept out of ``~/Downloads`` into
|
||||
``Media/<year>/<month>``, or one still in the pre-cloud ``keep/``.
|
||||
|
||||
Returns:
|
||||
list[Path]: Candidate directories; entries need not exist.
|
||||
"""
|
||||
media = cloud_root() / "Media"
|
||||
return [
|
||||
keep_dir(),
|
||||
*sorted(media.glob("*/wsg")),
|
||||
legacy_keep_dir(),
|
||||
downloads_dir(),
|
||||
*sorted(p for p in media.glob("*/*") if p.is_dir() and p.name != "wsg"),
|
||||
]
|
||||
|
||||
|
||||
def trash_dir() -> Path:
|
||||
"""Return the directory videos are moved to when passed.
|
||||
|
||||
Nothing ever removes files from here; clearing it is the user's call.
|
||||
Bounded to the newest ``TRASH_RETAIN`` passes by
|
||||
:mod:`python_pkg.wsg_grabber._prune`; older ones are deleted for good.
|
||||
|
||||
Returns:
|
||||
Path: ``<data dir>/trash``.
|
||||
|
||||
@ -28,6 +28,9 @@ class FileState(StrEnum):
|
||||
KEPT = "kept"
|
||||
PASSED = "passed"
|
||||
GONE = "gone"
|
||||
# A pass whose file has since been pruned from trash/: still known, so it
|
||||
# is never downloaded again, but no longer undoable.
|
||||
PURGED = "purged"
|
||||
|
||||
|
||||
class FileEvent(StrEnum):
|
||||
@ -45,7 +48,7 @@ class FileEvent(StrEnum):
|
||||
|
||||
|
||||
TERMINAL: Final[frozenset[FileState]] = frozenset(
|
||||
{FileState.KEPT, FileState.PASSED, FileState.GONE},
|
||||
{FileState.KEPT, FileState.PASSED, FileState.GONE, FileState.PURGED},
|
||||
)
|
||||
|
||||
CLAIMABLE: Final[frozenset[FileState]] = frozenset(
|
||||
|
||||
@ -165,3 +165,72 @@ def restore_for_review(conn: sqlite3.Connection, md5_b64: str, name: str) -> Non
|
||||
""",
|
||||
(FileState.READY.value, name, md5_b64),
|
||||
)
|
||||
|
||||
|
||||
def recent_passed_names(conn: sqlite3.Connection, limit: int) -> set[str]:
|
||||
"""Return the trash names of the *limit* most recent passes.
|
||||
|
||||
These are the files the trash prune keeps; everything else in ``trash/``
|
||||
goes.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
limit: How many passes to keep.
|
||||
|
||||
Returns:
|
||||
set[str]: Filenames inside ``trash/``.
|
||||
"""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT reviewed_name FROM files
|
||||
WHERE state = ? AND reviewed_name IS NOT NULL
|
||||
ORDER BY reviewed_at DESC, md5 DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(FileState.PASSED.value, limit),
|
||||
).fetchall()
|
||||
return {str(row["reviewed_name"]) for row in rows}
|
||||
|
||||
|
||||
def purge_older_passes(conn: sqlite3.Connection, limit: int) -> int:
|
||||
"""Mark every pass beyond the *limit* newest as purged.
|
||||
|
||||
A purged row keeps its md5, so the catalog never re-downloads the file,
|
||||
but drops out of the undo trail because its bytes are gone.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
limit: How many passes stay undoable.
|
||||
|
||||
Returns:
|
||||
int: Rows changed.
|
||||
"""
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
UPDATE files
|
||||
SET state = ?, reviewed_name = NULL
|
||||
WHERE state = ?
|
||||
AND md5 NOT IN (
|
||||
SELECT md5 FROM files
|
||||
WHERE state = ? AND reviewed_name IS NOT NULL
|
||||
ORDER BY reviewed_at DESC, md5 DESC
|
||||
LIMIT ?
|
||||
)
|
||||
""",
|
||||
(FileState.PURGED.value, FileState.PASSED.value, FileState.PASSED.value, limit),
|
||||
)
|
||||
return int(cursor.rowcount)
|
||||
|
||||
|
||||
def rename_kept(conn: sqlite3.Connection, old: str, new: str) -> None:
|
||||
"""Record that a kept file changed name while being migrated.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
old: Name the verdict recorded.
|
||||
new: Name the file has now.
|
||||
"""
|
||||
conn.execute(
|
||||
"UPDATE files SET reviewed_name = ? WHERE state = ? AND reviewed_name = ?",
|
||||
(new, FileState.KEPT.value, old),
|
||||
)
|
||||
|
||||
@ -94,13 +94,18 @@ def test_undo_puts_a_passed_file_back(session: app.Session) -> None:
|
||||
|
||||
|
||||
def test_undo_survives_a_collision_rename(session: app.Session) -> None:
|
||||
"""A verdict may have renamed the file; undo must follow the real name."""
|
||||
"""A verdict may have renamed the file; undo must follow the real name.
|
||||
|
||||
The stray ``clip.webm`` has no index row, so the prune that follows the
|
||||
pass deletes it -- which is exactly why the rename must never be undone
|
||||
by guessing the original name.
|
||||
"""
|
||||
_downloaded(session)
|
||||
(paths.trash_dir() / "clip.webm").write_bytes(b"older")
|
||||
state = session.commit(app.initial_state(session), Verdict.SKIP)
|
||||
assert (paths.trash_dir() / "clip-2.webm").exists()
|
||||
assert not (paths.trash_dir() / "clip.webm").exists()
|
||||
|
||||
state = session.undo(state)
|
||||
assert (paths.incoming_dir() / "clip.webm").read_bytes() == b"video"
|
||||
assert (paths.trash_dir() / "clip.webm").read_bytes() == b"older"
|
||||
assert not (paths.trash_dir() / "clip-2.webm").exists()
|
||||
|
||||
@ -45,9 +45,10 @@ def test_parser_exposes_the_subcommands() -> None:
|
||||
assert parsed.seconds == pytest.approx(5.0)
|
||||
|
||||
|
||||
def test_help_mentions_that_nothing_is_deleted() -> None:
|
||||
def test_help_says_where_keeps_and_passes_go() -> None:
|
||||
text = cli.build_parser().format_help()
|
||||
assert "Nothing is ever deleted" in text
|
||||
assert "cloud folder" in text
|
||||
assert "newest passes are kept" in text
|
||||
assert "trash" in text
|
||||
|
||||
|
||||
@ -76,7 +77,7 @@ def test_stats_reports_every_state(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
out = capsys.readouterr().out
|
||||
assert "known files 1" in out
|
||||
assert "new 1" in out
|
||||
assert "never emptied automatically" in out
|
||||
assert "newest 200 passes kept" in out
|
||||
|
||||
|
||||
def test_scrape_runs_and_shuts_down(
|
||||
|
||||
92
python_pkg/wsg_grabber/tests/test_housekeeping.py
Normal file
92
python_pkg/wsg_grabber/tests/test_housekeeping.py
Normal file
@ -0,0 +1,92 @@
|
||||
"""Tests for the around-a-verdict chores and for undo finding moved keeps."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from python_pkg.wsg_grabber import housekeeping, paths
|
||||
from python_pkg.wsg_grabber.models import Verdict
|
||||
from python_pkg.wsg_grabber.tests.conftest import item, reviewed
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prune_calls(monkeypatch: pytest.MonkeyPatch) -> list[object]:
|
||||
"""Replace the prune with a recorder.
|
||||
|
||||
Args:
|
||||
monkeypatch: Patching helper.
|
||||
|
||||
Returns:
|
||||
list[object]: One entry per prune call.
|
||||
"""
|
||||
calls: list[object] = []
|
||||
monkeypatch.setattr(
|
||||
housekeeping,
|
||||
"prune_trash",
|
||||
calls.append,
|
||||
)
|
||||
return calls
|
||||
|
||||
|
||||
def test_startup_migrates_then_prunes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
prune_calls: list[object],
|
||||
) -> None:
|
||||
order: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
housekeeping,
|
||||
"migrate_keep",
|
||||
lambda _conn: order.append("migrate"),
|
||||
)
|
||||
housekeeping.on_startup("conn")
|
||||
assert order == ["migrate"]
|
||||
assert prune_calls == ["conn"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("choice", "expected"),
|
||||
[(Verdict.SKIP, ["conn"]), (Verdict.KEEP, [])],
|
||||
)
|
||||
def test_only_a_pass_triggers_a_prune(
|
||||
prune_calls: list[object],
|
||||
choice: Verdict,
|
||||
expected: list[object],
|
||||
) -> None:
|
||||
housekeeping.after_verdict("conn", choice)
|
||||
assert prune_calls == expected
|
||||
|
||||
|
||||
def test_a_pass_is_located_in_trash() -> None:
|
||||
action = reviewed(item(), Verdict.SKIP)
|
||||
assert housekeeping.locate_reviewed(action) == paths.trash_dir() / "a.webm"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"place",
|
||||
[
|
||||
paths.keep_dir,
|
||||
lambda: paths.cloud_keep_dir(2001),
|
||||
paths.legacy_keep_dir,
|
||||
paths.downloads_dir,
|
||||
lambda: paths.cloud_root() / "Media" / "2001" / "03",
|
||||
],
|
||||
)
|
||||
def test_a_keep_is_found_wherever_it_ended_up(place: Callable[[], Path]) -> None:
|
||||
paths.cloud_root().mkdir(parents=True)
|
||||
directory = place()
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
(directory / "a.webm").write_bytes(b"v")
|
||||
|
||||
action = reviewed(item(), Verdict.KEEP)
|
||||
assert housekeeping.locate_reviewed(action) == directory / "a.webm"
|
||||
|
||||
|
||||
def test_a_keep_that_is_nowhere_points_at_the_current_keep_dir() -> None:
|
||||
action = reviewed(item(), Verdict.KEEP)
|
||||
assert housekeeping.locate_reviewed(action) == paths.keep_dir() / "a.webm"
|
||||
130
python_pkg/wsg_grabber/tests/test_migrate_keep.py
Normal file
130
python_pkg/wsg_grabber/tests/test_migrate_keep.py
Normal file
@ -0,0 +1,130 @@
|
||||
"""Tests for moving kept videos out of the data dir."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from python_pkg.wsg_grabber import db, paths, store, store_verdicts
|
||||
from python_pkg.wsg_grabber._migrate_keep import migrate_keep
|
||||
from python_pkg.wsg_grabber.models import RemoteFile
|
||||
from python_pkg.wsg_grabber.states import FileState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
import sqlite3
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def conn() -> Iterator[sqlite3.Connection]:
|
||||
"""Open a sandboxed index.
|
||||
|
||||
Yields:
|
||||
sqlite3.Connection: Closed afterwards.
|
||||
"""
|
||||
paths.ensure_dirs()
|
||||
opened = db.open_index(paths.db_path())
|
||||
try:
|
||||
yield opened
|
||||
finally:
|
||||
opened.close()
|
||||
|
||||
|
||||
def _kept(conn: sqlite3.Connection, name: str, payload: bytes) -> None:
|
||||
"""Record a keep whose file sits in the legacy keep/.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
name: Filename.
|
||||
payload: File contents.
|
||||
"""
|
||||
store.record_files(
|
||||
conn,
|
||||
[
|
||||
RemoteFile(
|
||||
md5=f"md5-{name}",
|
||||
tim=1,
|
||||
ext=".webm",
|
||||
orig_name=name,
|
||||
fsize=len(payload),
|
||||
width=1,
|
||||
height=1,
|
||||
thread_no=1,
|
||||
post_no=1,
|
||||
),
|
||||
],
|
||||
)
|
||||
store.mark_downloaded(conn, f"md5-{name}", name)
|
||||
store_verdicts.record_verdict(conn, f"md5-{name}", FileState.KEPT, name)
|
||||
paths.legacy_keep_dir().mkdir(parents=True, exist_ok=True)
|
||||
(paths.legacy_keep_dir() / name).write_bytes(payload)
|
||||
|
||||
|
||||
def test_moves_everything_and_removes_the_empty_legacy_dir(
|
||||
conn: sqlite3.Connection,
|
||||
) -> None:
|
||||
_kept(conn, "a.webm", b"aa")
|
||||
_kept(conn, "b.webm", b"bb")
|
||||
|
||||
assert migrate_keep(conn) == 2
|
||||
|
||||
assert (paths.keep_dir() / "a.webm").read_bytes() == b"aa"
|
||||
assert (paths.keep_dir() / "b.webm").read_bytes() == b"bb"
|
||||
assert not paths.legacy_keep_dir().exists()
|
||||
|
||||
|
||||
def test_no_legacy_dir_is_a_no_op(conn: sqlite3.Connection) -> None:
|
||||
assert not paths.legacy_keep_dir().exists()
|
||||
assert migrate_keep(conn) == 0
|
||||
|
||||
|
||||
def test_identical_copy_at_destination_drops_the_legacy_file(
|
||||
conn: sqlite3.Connection,
|
||||
) -> None:
|
||||
_kept(conn, "a.webm", b"aa")
|
||||
paths.keep_dir().mkdir(parents=True, exist_ok=True)
|
||||
(paths.keep_dir() / "a.webm").write_bytes(b"zz")
|
||||
|
||||
assert migrate_keep(conn) == 0
|
||||
|
||||
assert (paths.keep_dir() / "a.webm").read_bytes() == b"zz"
|
||||
assert not paths.legacy_keep_dir().exists()
|
||||
|
||||
|
||||
def test_name_clash_with_different_size_is_suffixed_and_reindexed(
|
||||
conn: sqlite3.Connection,
|
||||
) -> None:
|
||||
_kept(conn, "a.webm", b"aa")
|
||||
paths.keep_dir().mkdir(parents=True, exist_ok=True)
|
||||
(paths.keep_dir() / "a.webm").write_bytes(b"other")
|
||||
|
||||
assert migrate_keep(conn) == 1
|
||||
|
||||
assert (paths.keep_dir() / "a.webm").read_bytes() == b"other"
|
||||
assert (paths.keep_dir() / "a-2.webm").read_bytes() == b"aa"
|
||||
row = conn.execute(
|
||||
"SELECT reviewed_name FROM files WHERE md5 = ?",
|
||||
("md5-a.webm",),
|
||||
).fetchone()
|
||||
assert row["reviewed_name"] == "a-2.webm"
|
||||
|
||||
|
||||
def test_subdirectories_are_left_alone(conn: sqlite3.Connection) -> None:
|
||||
paths.legacy_keep_dir().mkdir(parents=True)
|
||||
(paths.legacy_keep_dir() / "nested").mkdir()
|
||||
|
||||
assert migrate_keep(conn) == 0
|
||||
assert (paths.legacy_keep_dir() / "nested").is_dir()
|
||||
|
||||
|
||||
def test_a_file_that_vanishes_mid_move_is_skipped(
|
||||
conn: sqlite3.Connection,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_kept(conn, "a.webm", b"aa")
|
||||
monkeypatch.setattr(
|
||||
"python_pkg.wsg_grabber._migrate_keep.files.apply_move",
|
||||
lambda _move: False,
|
||||
)
|
||||
assert migrate_keep(conn) == 0
|
||||
@ -35,13 +35,42 @@ def test_derived_paths_live_under_the_data_dir() -> None:
|
||||
for candidate in (
|
||||
paths.db_path(),
|
||||
paths.incoming_dir(),
|
||||
paths.keep_dir(),
|
||||
paths.legacy_keep_dir(),
|
||||
paths.trash_dir(),
|
||||
paths.ipc_socket_path(),
|
||||
):
|
||||
assert candidate.parent == root
|
||||
|
||||
|
||||
def test_keep_dir_falls_back_to_downloads_without_the_cloud_folder() -> None:
|
||||
assert not paths.cloud_root().exists()
|
||||
assert paths.keep_dir() == Path.home() / "Downloads"
|
||||
|
||||
|
||||
def test_keep_dir_uses_the_cloud_media_folder_for_this_year() -> None:
|
||||
paths.cloud_root().mkdir(parents=True)
|
||||
kept = paths.keep_dir()
|
||||
assert kept.parent.parent == paths.cloud_root() / "Media"
|
||||
assert kept.name == "wsg"
|
||||
assert kept.parent.name.isdigit()
|
||||
assert kept == paths.cloud_keep_dir(int(kept.parent.name))
|
||||
|
||||
|
||||
def test_kept_candidates_cover_every_place_a_keep_can_end_up() -> None:
|
||||
media = paths.cloud_root() / "Media"
|
||||
for sub in ("2025/wsg", "2026/wsg", "2026/09", "2026/notes.txt"):
|
||||
(media / sub).parent.mkdir(parents=True, exist_ok=True)
|
||||
(media / "2025/wsg").mkdir(parents=True)
|
||||
(media / "2026/wsg").mkdir(parents=True)
|
||||
(media / "2026/09").mkdir(parents=True)
|
||||
(media / "2026/notes.txt").write_text("")
|
||||
candidates = paths.kept_candidates()
|
||||
assert candidates[0] == paths.keep_dir()
|
||||
assert candidates[1:3] == [media / "2025/wsg", media / "2026/wsg"]
|
||||
assert candidates[3:5] == [paths.legacy_keep_dir(), paths.downloads_dir()]
|
||||
assert candidates[5:] == [media / "2026/09"]
|
||||
|
||||
|
||||
def test_socket_path_names_the_board() -> None:
|
||||
assert paths.ipc_socket_path().name == "mpv-wsg.sock"
|
||||
|
||||
|
||||
127
python_pkg/wsg_grabber/tests/test_prune.py
Normal file
127
python_pkg/wsg_grabber/tests/test_prune.py
Normal file
@ -0,0 +1,127 @@
|
||||
"""Tests for the trash cap."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from python_pkg.wsg_grabber import db, files, paths, store, store_verdicts
|
||||
from python_pkg.wsg_grabber._prune import prune_trash
|
||||
from python_pkg.wsg_grabber.models import RemoteFile
|
||||
from python_pkg.wsg_grabber.states import FileState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
import sqlite3
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def conn() -> Iterator[sqlite3.Connection]:
|
||||
"""Open a sandboxed index with the on-disk layout in place.
|
||||
|
||||
Yields:
|
||||
sqlite3.Connection: Closed afterwards.
|
||||
"""
|
||||
paths.ensure_dirs()
|
||||
opened = db.open_index(paths.db_path())
|
||||
try:
|
||||
yield opened
|
||||
finally:
|
||||
opened.close()
|
||||
|
||||
|
||||
def _passed(conn: sqlite3.Connection, index: int, reviewed_at: str) -> str:
|
||||
"""Record one pass whose file sits in trash/.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
index: Distinguishes files within a test.
|
||||
reviewed_at: Verdict timestamp, so ordering is explicit.
|
||||
|
||||
Returns:
|
||||
str: Name of the trashed file.
|
||||
"""
|
||||
md5 = f"md5-{index:04d}"
|
||||
name = f"{index:04d}.webm"
|
||||
store.record_files(
|
||||
conn,
|
||||
[
|
||||
RemoteFile(
|
||||
md5=md5,
|
||||
tim=index,
|
||||
ext=".webm",
|
||||
orig_name=name,
|
||||
fsize=1,
|
||||
width=1,
|
||||
height=1,
|
||||
thread_no=1,
|
||||
post_no=index,
|
||||
),
|
||||
],
|
||||
)
|
||||
store.mark_downloaded(conn, md5, name)
|
||||
store_verdicts.record_verdict(conn, md5, FileState.PASSED, name)
|
||||
conn.execute("UPDATE files SET reviewed_at = ? WHERE md5 = ?", (reviewed_at, md5))
|
||||
(paths.trash_dir() / name).write_bytes(b"v")
|
||||
return name
|
||||
|
||||
|
||||
def test_keeps_only_the_newest_passes(conn: sqlite3.Connection) -> None:
|
||||
names = [_passed(conn, i, f"2026-09-{i + 1:02d}") for i in range(5)]
|
||||
|
||||
assert prune_trash(conn, retain=2) == 3
|
||||
|
||||
assert sorted(p.name for p in paths.trash_dir().iterdir()) == names[3:]
|
||||
assert store.state_of(conn, "md5-0000") is FileState.PURGED
|
||||
assert store.state_of(conn, "md5-0004") is FileState.PASSED
|
||||
assert store_verdicts.reviewed_count(conn) == 2
|
||||
|
||||
|
||||
def test_orphans_without_an_index_row_are_deleted_too(
|
||||
conn: sqlite3.Connection,
|
||||
) -> None:
|
||||
_passed(conn, 1, "2026-09-01")
|
||||
(paths.trash_dir() / "orphan.webm").write_bytes(b"?")
|
||||
(paths.trash_dir() / "subdir").mkdir()
|
||||
|
||||
assert prune_trash(conn, retain=5) == 1
|
||||
|
||||
assert not (paths.trash_dir() / "orphan.webm").exists()
|
||||
assert (paths.trash_dir() / "0001.webm").exists()
|
||||
assert (paths.trash_dir() / "subdir").is_dir()
|
||||
|
||||
|
||||
def test_under_the_cap_is_a_no_op(conn: sqlite3.Connection) -> None:
|
||||
_passed(conn, 1, "2026-09-01")
|
||||
assert prune_trash(conn, retain=5) == 0
|
||||
assert store.state_of(conn, "md5-0001") is FileState.PASSED
|
||||
|
||||
|
||||
def test_missing_trash_dir_is_tolerated(conn: sqlite3.Connection) -> None:
|
||||
paths.trash_dir().rmdir()
|
||||
assert prune_trash(conn) == 0
|
||||
|
||||
|
||||
def test_a_pass_whose_file_was_forgotten_is_purged(conn: sqlite3.Connection) -> None:
|
||||
"""A row with no reviewed_name cannot be undone, so it is written off."""
|
||||
_passed(conn, 1, "2026-09-01")
|
||||
store_verdicts.forget_verdict(conn, "md5-0001")
|
||||
|
||||
prune_trash(conn, retain=5)
|
||||
|
||||
assert store.state_of(conn, "md5-0001") is FileState.PURGED
|
||||
|
||||
|
||||
def test_purged_rows_are_still_known_to_the_catalog(conn: sqlite3.Connection) -> None:
|
||||
_passed(conn, 1, "2026-09-01")
|
||||
prune_trash(conn, retain=0)
|
||||
assert "md5-0001" in store.known_md5s(conn)
|
||||
|
||||
|
||||
def test_remove_reports_whether_anything_was_there() -> None:
|
||||
target = paths.data_dir() / "x"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(b"")
|
||||
assert files.remove(target) is True
|
||||
assert files.remove(target) is False
|
||||
@ -80,8 +80,13 @@ def test_illegal_transition_from_a_terminal_state_raises() -> None:
|
||||
next_state(FileState.GONE, FileEvent.CLAIMED, 0)
|
||||
|
||||
|
||||
def test_terminal_states_are_exactly_the_three_end_points() -> None:
|
||||
assert {FileState.KEPT, FileState.PASSED, FileState.GONE} == TERMINAL
|
||||
def test_terminal_states_are_exactly_the_four_end_points() -> None:
|
||||
assert {
|
||||
FileState.KEPT,
|
||||
FileState.PASSED,
|
||||
FileState.GONE,
|
||||
FileState.PURGED,
|
||||
} == TERMINAL
|
||||
for state in TERMINAL:
|
||||
assert is_terminal(state)
|
||||
for state in CLAIMABLE:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user