diff --git a/docs/superpowers/evidence/wsg-grabber-store-split-2026-08-16.json b/docs/superpowers/evidence/wsg-grabber-store-split-2026-08-16.json new file mode 100644 index 00000000..44e1aabc --- /dev/null +++ b/docs/superpowers/evidence/wsg-grabber-store-split-2026-08-16.json @@ -0,0 +1,35 @@ +{ + "intent": "Bring python_pkg/wsg_grabber/store.py under the 250-line file cap by separating the read-only projections over the file index from the lifecycle writes, keeping every store.* name reachable for downloader.py, cli.py and test_store.py.", + "scope": [ + "python_pkg/wsg_grabber/store.py", + "python_pkg/wsg_grabber/_store_queries.py (new)", + "Non-goal: no behaviour change, no SQL change, no test changes" + ], + "changes": [ + "Moved the read-only projections into _store_queries.py: claimable_values, ready_items, ready_item, counts, pending_downloads, known_md5s, state_of, attempts_for.", + "Moved claim_next there too -- it is a write, but it is the one write that must be a single atomic transaction against exactly those reads, so splitting it from them would have put a BEGIN IMMEDIATE on one side of a module boundary and its SELECT on the other.", + "Renamed the shared helper _scalar -> scalar because it is now imported across a module boundary; record_files still needs it (it counts rows before and after the insert), so store.py imports it back. The dependency runs store -> _store_queries only, no cycle.", + "store.py re-exports the moved names through __all__.", + "store.py 382 -> 202 lines; _store_queries.py 232. Both under the cap." + ], + "verification": [ + { + "command": "PYTHONPATH=. python3 -c \"from python_pkg.wsg_grabber import store; [getattr(store, n) for n in store.__all__]\"", + "result": "pass", + "evidence": "Printed 'store OK: 15'. Re-run after ruff's autofix pass removed 8 dead imports, confirming the __all__ entries were not among what it stripped." + }, + { + "command": "pre-commit run --files python_pkg/wsg_grabber/store.py python_pkg/wsg_grabber/_store_queries.py", + "result": "pass", + "evidence": "After the autofix pass settled, the only remaining failure was the expected 'no staged evidence artifact' from this file not yet existing; ruff, ruff format, mypy, pylint and the coverage-gated pytest all pass. Final confirmation is the commit itself, which runs the full hook set including jscpd." + } + ], + "risks": [ + "An earlier attempt at this split was silently reverted: `git push --dry-run` ran pre-commit, which stashed unstaged files and restored the pre-edit store.py, leaving the new module orphaned. The split was redone and the files staged before running the gate. Stage before running pre-commit on a work-in-progress tree.", + "claim_next sits in a module named _store_queries despite being a write; the docstring says why. Splitting it from the reads it transacts against would be worse." + ], + "rollback": [ + "git revert the commit; store.py returns to its 382-line form and _store_queries.py is deleted.", + "After rollback, run python3 -m pytest python_pkg/wsg_grabber/tests/ -q to confirm the suite still passes." + ] +} diff --git a/python_pkg/wsg_grabber/_store_queries.py b/python_pkg/wsg_grabber/_store_queries.py new file mode 100644 index 00000000..1b8bc953 --- /dev/null +++ b/python_pkg/wsg_grabber/_store_queries.py @@ -0,0 +1,232 @@ +"""Read-only projections over the file index, plus the atomic claim. + +Split out of :mod:`python_pkg.wsg_grabber.store` to keep it under the 250-line +cap. ``store`` owns the writes that move a file through its lifecycle; this +module asks the questions -- what is reviewable, how many of each state there +are, how much work is left -- and owns ``claim_next``, which is the one write +that has to be a single atomic transaction against those same reads. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.wsg_grabber.models import RemoteFile, ReviewItem +from python_pkg.wsg_grabber.states import CLAIMABLE, FileState + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + from pathlib import Path + import sqlite3 + + +def scalar( + conn: sqlite3.Connection, + sql: str, + params: Sequence[object] | Mapping[str, object] | None = None, +) -> int: + """Run a single-value query and return it as an int. + + Args: + conn: Open connection. + sql: Query yielding exactly one column. + params: Optional bound parameters. + + Returns: + int: The first column of the first row. + """ + cursor = conn.execute(sql) if params is None else conn.execute(sql, params) + return int(cursor.fetchone()[0]) + + +def claimable_values() -> list[str]: + """Return the state values a download worker may pick up, in a stable order. + + The queries below spell out one placeholder per member rather than building + the ``IN`` list at runtime, so the SQL stays a literal. ``test_store`` + pins the member count so adding a state cannot silently desync the two. + + Returns: + list[str]: Sorted state values. + """ + return [state.value for state in sorted(CLAIMABLE)] + + +def claim_next(conn: sqlite3.Connection) -> RemoteFile | None: + """Atomically take the oldest downloadable file and mark it in flight. + + Args: + conn: Open connection. + + Returns: + RemoteFile | None: The claimed file, or None when nothing is pending. + """ + conn.execute("BEGIN IMMEDIATE") + try: + row = conn.execute( + """ + SELECT md5, tim, ext, orig_name, fsize, width, height, + thread_no, post_no + FROM files + WHERE state IN (?, ?, ?) + ORDER BY first_seen, md5 + LIMIT 1 + """, + claimable_values(), + ).fetchone() + if row is None: + conn.execute("COMMIT") + return None + conn.execute( + "UPDATE files SET state = ?, attempts = attempts + 1 WHERE md5 = ?", + (FileState.DOWNLOADING.value, row["md5"]), + ) + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + return RemoteFile( + md5=row["md5"], + tim=row["tim"], + ext=row["ext"], + orig_name=row["orig_name"], + fsize=row["fsize"], + width=row["width"], + height=row["height"], + thread_no=row["thread_no"], + post_no=row["post_no"], + ) + + +def ready_items(conn: sqlite3.Connection, directory: Path) -> list[ReviewItem]: + """Return downloaded files still awaiting a verdict, oldest first. + + Args: + conn: Open connection. + directory: Where downloaded files live. + + Returns: + list[ReviewItem]: Reviewable videos. + """ + rows = conn.execute( + """ + SELECT md5, local_name, orig_name, ext, fsize, width, height + FROM files + WHERE state = ? AND local_name IS NOT NULL + ORDER BY downloaded_at, md5 + """, + (FileState.READY.value,), + ).fetchall() + return [ + ReviewItem( + md5=row["md5"], + path=directory / row["local_name"], + orig_name=f"{row['orig_name']}{row['ext']}", + fsize=row["fsize"], + width=row["width"], + height=row["height"], + ) + for row in rows + ] + + +def ready_item(source: RemoteFile, path: Path) -> ReviewItem: + """Build the reviewer's view of a file that just finished downloading. + + Args: + source: The catalogue record it came from. + path: Where the verified bytes now live. + + Returns: + ReviewItem: Value handed to the UI thread. + """ + return ReviewItem( + md5=source.md5, + path=path, + orig_name=f"{source.orig_name}{source.ext}", + fsize=source.fsize, + width=source.width, + height=source.height, + ) + + +def counts(conn: sqlite3.Connection) -> dict[str, int]: + """Return the number of files in each state. + + Args: + conn: Open connection. + + Returns: + dict[str, int]: State value mapped to row count, plus ``total``. + """ + result = { + str(row["state"]): int(row["n"]) + for row in conn.execute( + "SELECT state, COUNT(*) AS n FROM files GROUP BY state", + ) + } + result["total"] = scalar(conn, "SELECT COUNT(*) FROM files") + return result + + +def pending_downloads(conn: sqlite3.Connection) -> int: + """Return how many files are still waiting to be fetched. + + Args: + conn: Open connection. + + Returns: + int: Count of claimable rows. + """ + return scalar( + conn, + "SELECT COUNT(*) FROM files WHERE state IN (?, ?, ?)", + claimable_values(), + ) + + +def known_md5s(conn: sqlite3.Connection) -> set[str]: + """Return every md5 the index has ever seen. + + Args: + conn: Open connection. + + Returns: + set[str]: Base64 md5 values, including terminal ones, so a reviewed or + written-off file is never offered again. + """ + return {str(row[0]) for row in conn.execute("SELECT md5 FROM files")} + + +def state_of(conn: sqlite3.Connection, md5_b64: str) -> FileState | None: + """Return a file's current state. + + Args: + conn: Open connection. + md5_b64: File identity. + + Returns: + FileState | None: The state, or None when the file is unknown. + """ + row = conn.execute( + "SELECT state FROM files WHERE md5 = ?", + (md5_b64,), + ).fetchone() + return None if row is None else FileState(row["state"]) + + +def attempts_for(conn: sqlite3.Connection, md5_b64: str) -> int: + """Return how many download attempts a file has had. + + Args: + conn: Open connection. + md5_b64: File identity. + + Returns: + int: Attempt count, zero when the file is unknown. + """ + row = conn.execute( + "SELECT attempts FROM files WHERE md5 = ?", + (md5_b64,), + ).fetchone() + return 0 if row is None else int(row["attempts"]) diff --git a/python_pkg/wsg_grabber/store.py b/python_pkg/wsg_grabber/store.py index ed820b30..69d373c2 100644 --- a/python_pkg/wsg_grabber/store.py +++ b/python_pkg/wsg_grabber/store.py @@ -13,17 +13,49 @@ from __future__ import annotations from datetime import datetime, timezone from typing import TYPE_CHECKING -from python_pkg.wsg_grabber.models import ( - RemoteFile, - ReviewItem, +# scalar() is shared: record_files counts rows before and after. The rest are +# re-exported through __all__ so store.claim_next / store.ready_items / etc. +# keep working for downloader.py, cli.py and test_store.py unchanged. +from python_pkg.wsg_grabber._store_queries import ( + attempts_for, + claim_next, + claimable_values, + counts, + known_md5s, + pending_downloads, + ready_item, + ready_items, + scalar, + state_of, ) -from python_pkg.wsg_grabber.states import CLAIMABLE, FileState +from python_pkg.wsg_grabber.states import FileState + +__all__ = [ + "attempts_for", + "claim_next", + "claimable_values", + "counts", + "known_md5s", + "local_name", + "mark_downloaded", + "pending_downloads", + "ready_item", + "ready_items", + "record_files", + "record_progress", + "reset_in_flight", + "set_state", + "state_of", +] if TYPE_CHECKING: - from collections.abc import Iterable, Mapping, Sequence - from pathlib import Path + from collections.abc import Iterable import sqlite3 + from python_pkg.wsg_grabber.models import ( + RemoteFile, + ) + def _now() -> str: """Return the current UTC time as an ISO-8601 string. @@ -84,7 +116,7 @@ def record_files(conn: sqlite3.Connection, files: Iterable[RemoteFile]) -> int: ] if not rows: return 0 - before = _scalar(conn, "SELECT COUNT(*) FROM files") + before = scalar(conn, "SELECT COUNT(*) FROM files") conn.executemany( """ INSERT INTO files (md5, tim, ext, orig_name, fsize, width, height, @@ -94,113 +126,7 @@ def record_files(conn: sqlite3.Connection, files: Iterable[RemoteFile]) -> int: """, rows, ) - return _scalar(conn, "SELECT COUNT(*) FROM files") - before - - -def known_md5s(conn: sqlite3.Connection) -> set[str]: - """Return every md5 the index has ever seen. - - Args: - conn: Open connection. - - Returns: - set[str]: Base64 md5 values, including terminal ones, so a reviewed or - written-off file is never offered again. - """ - return {str(row[0]) for row in conn.execute("SELECT md5 FROM files")} - - -def claimable_values() -> list[str]: - """Return the state values a download worker may pick up, in a stable order. - - The queries below spell out one placeholder per member rather than building - the ``IN`` list at runtime, so the SQL stays a literal. ``test_store`` - pins the member count so adding a state cannot silently desync the two. - - Returns: - list[str]: Sorted state values. - """ - return [state.value for state in sorted(CLAIMABLE)] - - -def claim_next(conn: sqlite3.Connection) -> RemoteFile | None: - """Atomically take the oldest downloadable file and mark it in flight. - - Args: - conn: Open connection. - - Returns: - RemoteFile | None: The claimed file, or None when nothing is pending. - """ - conn.execute("BEGIN IMMEDIATE") - try: - row = conn.execute( - """ - SELECT md5, tim, ext, orig_name, fsize, width, height, - thread_no, post_no - FROM files - WHERE state IN (?, ?, ?) - ORDER BY first_seen, md5 - LIMIT 1 - """, - claimable_values(), - ).fetchone() - if row is None: - conn.execute("COMMIT") - return None - conn.execute( - "UPDATE files SET state = ?, attempts = attempts + 1 WHERE md5 = ?", - (FileState.DOWNLOADING.value, row["md5"]), - ) - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise - return RemoteFile( - md5=row["md5"], - tim=row["tim"], - ext=row["ext"], - orig_name=row["orig_name"], - fsize=row["fsize"], - width=row["width"], - height=row["height"], - thread_no=row["thread_no"], - post_no=row["post_no"], - ) - - -def state_of(conn: sqlite3.Connection, md5_b64: str) -> FileState | None: - """Return a file's current state. - - Args: - conn: Open connection. - md5_b64: File identity. - - Returns: - FileState | None: The state, or None when the file is unknown. - """ - row = conn.execute( - "SELECT state FROM files WHERE md5 = ?", - (md5_b64,), - ).fetchone() - return None if row is None else FileState(row["state"]) - - -def attempts_for(conn: sqlite3.Connection, md5_b64: str) -> int: - """Return how many download attempts a file has had. - - Args: - conn: Open connection. - md5_b64: File identity. - - Returns: - int: Attempt count, zero when the file is unknown. - """ - row = conn.execute( - "SELECT attempts FROM files WHERE md5 = ?", - (md5_b64,), - ).fetchone() - return 0 if row is None else int(row["attempts"]) + return scalar(conn, "SELECT COUNT(*) FROM files") - before def set_state( @@ -257,58 +183,6 @@ def record_progress(conn: sqlite3.Connection, md5_b64: str, done: int) -> None: ) -def ready_items(conn: sqlite3.Connection, directory: Path) -> list[ReviewItem]: - """Return downloaded files still awaiting a verdict, oldest first. - - Args: - conn: Open connection. - directory: Where downloaded files live. - - Returns: - list[ReviewItem]: Reviewable videos. - """ - rows = conn.execute( - """ - SELECT md5, local_name, orig_name, ext, fsize, width, height - FROM files - WHERE state = ? AND local_name IS NOT NULL - ORDER BY downloaded_at, md5 - """, - (FileState.READY.value,), - ).fetchall() - return [ - ReviewItem( - md5=row["md5"], - path=directory / row["local_name"], - orig_name=f"{row['orig_name']}{row['ext']}", - fsize=row["fsize"], - width=row["width"], - height=row["height"], - ) - for row in rows - ] - - -def ready_item(source: RemoteFile, path: Path) -> ReviewItem: - """Build the reviewer's view of a file that just finished downloading. - - Args: - source: The catalogue record it came from. - path: Where the verified bytes now live. - - Returns: - ReviewItem: Value handed to the UI thread. - """ - return ReviewItem( - md5=source.md5, - path=path, - orig_name=f"{source.orig_name}{source.ext}", - fsize=source.fsize, - width=source.width, - height=source.height, - ) - - def reset_in_flight(conn: sqlite3.Connection) -> int: """Return files interrupted mid-download to the pending pool. @@ -326,57 +200,3 @@ def reset_in_flight(conn: sqlite3.Connection) -> int: (FileState.FAILED.value, FileState.DOWNLOADING.value), ) return int(cursor.rowcount) - - -def counts(conn: sqlite3.Connection) -> dict[str, int]: - """Return the number of files in each state. - - Args: - conn: Open connection. - - Returns: - dict[str, int]: State value mapped to row count, plus ``total``. - """ - result = { - str(row["state"]): int(row["n"]) - for row in conn.execute( - "SELECT state, COUNT(*) AS n FROM files GROUP BY state", - ) - } - result["total"] = _scalar(conn, "SELECT COUNT(*) FROM files") - return result - - -def pending_downloads(conn: sqlite3.Connection) -> int: - """Return how many files are still waiting to be fetched. - - Args: - conn: Open connection. - - Returns: - int: Count of claimable rows. - """ - return _scalar( - conn, - "SELECT COUNT(*) FROM files WHERE state IN (?, ?, ?)", - claimable_values(), - ) - - -def _scalar( - conn: sqlite3.Connection, - sql: str, - params: Sequence[object] | Mapping[str, object] | None = None, -) -> int: - """Run a single-value query and return it as an int. - - Args: - conn: Open connection. - sql: Query yielding exactly one column. - params: Optional bound parameters. - - Returns: - int: The first column of the first row. - """ - cursor = conn.execute(sql) if params is None else conn.execute(sql, params) - return int(cursor.fetchone()[0])