mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-19 01:54:21 +01:00
Split wsg_grabber/store.py under the 250-line cap
store.py mixed the writes that move a file through its lifecycle with the read-only projections over the same table. The reads move to _store_queries.py: claimable_values, ready_items, ready_item, counts, pending_downloads, known_md5s, state_of, attempts_for. claim_next goes with them despite being a write. It is the one write that has to be a single atomic transaction against exactly those reads, so splitting it from them would have left BEGIN IMMEDIATE on one side of a module boundary and its SELECT on the other. _scalar becomes scalar: record_files still counts rows before and after its insert, so store imports it back. The dependency runs one way only. store.py re-exports through __all__, so downloader.py, cli.py and test_store.py keep reaching everything at store.* unchanged -- all 15 names verified to resolve after ruff's autofix pass. store.py 382 -> 202, _store_queries.py 232. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011w2Pj4Ek3ZNMQVYcFHgXDY
This commit is contained in:
parent
262ef38efe
commit
f58caea2ab
@ -0,0 +1,35 @@
|
||||
{
|
||||
"intent": "Bring python_pkg/wsg_grabber/store.py under the 250-line file cap by separating the read-only projections over the file index from the lifecycle writes, keeping every store.* name reachable for downloader.py, cli.py and test_store.py.",
|
||||
"scope": [
|
||||
"python_pkg/wsg_grabber/store.py",
|
||||
"python_pkg/wsg_grabber/_store_queries.py (new)",
|
||||
"Non-goal: no behaviour change, no SQL change, no test changes"
|
||||
],
|
||||
"changes": [
|
||||
"Moved the read-only projections into _store_queries.py: claimable_values, ready_items, ready_item, counts, pending_downloads, known_md5s, state_of, attempts_for.",
|
||||
"Moved claim_next there too -- it is a write, but it is the one write that must be a single atomic transaction against exactly those reads, so splitting it from them would have put a BEGIN IMMEDIATE on one side of a module boundary and its SELECT on the other.",
|
||||
"Renamed the shared helper _scalar -> scalar because it is now imported across a module boundary; record_files still needs it (it counts rows before and after the insert), so store.py imports it back. The dependency runs store -> _store_queries only, no cycle.",
|
||||
"store.py re-exports the moved names through __all__.",
|
||||
"store.py 382 -> 202 lines; _store_queries.py 232. Both under the cap."
|
||||
],
|
||||
"verification": [
|
||||
{
|
||||
"command": "PYTHONPATH=. python3 -c \"from python_pkg.wsg_grabber import store; [getattr(store, n) for n in store.__all__]\"",
|
||||
"result": "pass",
|
||||
"evidence": "Printed 'store OK: 15'. Re-run after ruff's autofix pass removed 8 dead imports, confirming the __all__ entries were not among what it stripped."
|
||||
},
|
||||
{
|
||||
"command": "pre-commit run --files python_pkg/wsg_grabber/store.py python_pkg/wsg_grabber/_store_queries.py",
|
||||
"result": "pass",
|
||||
"evidence": "After the autofix pass settled, the only remaining failure was the expected 'no staged evidence artifact' from this file not yet existing; ruff, ruff format, mypy, pylint and the coverage-gated pytest all pass. Final confirmation is the commit itself, which runs the full hook set including jscpd."
|
||||
}
|
||||
],
|
||||
"risks": [
|
||||
"An earlier attempt at this split was silently reverted: `git push --dry-run` ran pre-commit, which stashed unstaged files and restored the pre-edit store.py, leaving the new module orphaned. The split was redone and the files staged before running the gate. Stage before running pre-commit on a work-in-progress tree.",
|
||||
"claim_next sits in a module named _store_queries despite being a write; the docstring says why. Splitting it from the reads it transacts against would be worse."
|
||||
],
|
||||
"rollback": [
|
||||
"git revert the commit; store.py returns to its 382-line form and _store_queries.py is deleted.",
|
||||
"After rollback, run python3 -m pytest python_pkg/wsg_grabber/tests/ -q to confirm the suite still passes."
|
||||
]
|
||||
}
|
||||
232
python_pkg/wsg_grabber/_store_queries.py
Normal file
232
python_pkg/wsg_grabber/_store_queries.py
Normal file
@ -0,0 +1,232 @@
|
||||
"""Read-only projections over the file index, plus the atomic claim.
|
||||
|
||||
Split out of :mod:`python_pkg.wsg_grabber.store` to keep it under the 250-line
|
||||
cap. ``store`` owns the writes that move a file through its lifecycle; this
|
||||
module asks the questions -- what is reviewable, how many of each state there
|
||||
are, how much work is left -- and owns ``claim_next``, which is the one write
|
||||
that has to be a single atomic transaction against those same reads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python_pkg.wsg_grabber.models import RemoteFile, ReviewItem
|
||||
from python_pkg.wsg_grabber.states import CLAIMABLE, FileState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
|
||||
|
||||
def scalar(
|
||||
conn: sqlite3.Connection,
|
||||
sql: str,
|
||||
params: Sequence[object] | Mapping[str, object] | None = None,
|
||||
) -> int:
|
||||
"""Run a single-value query and return it as an int.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
sql: Query yielding exactly one column.
|
||||
params: Optional bound parameters.
|
||||
|
||||
Returns:
|
||||
int: The first column of the first row.
|
||||
"""
|
||||
cursor = conn.execute(sql) if params is None else conn.execute(sql, params)
|
||||
return int(cursor.fetchone()[0])
|
||||
|
||||
|
||||
def claimable_values() -> list[str]:
|
||||
"""Return the state values a download worker may pick up, in a stable order.
|
||||
|
||||
The queries below spell out one placeholder per member rather than building
|
||||
the ``IN`` list at runtime, so the SQL stays a literal. ``test_store``
|
||||
pins the member count so adding a state cannot silently desync the two.
|
||||
|
||||
Returns:
|
||||
list[str]: Sorted state values.
|
||||
"""
|
||||
return [state.value for state in sorted(CLAIMABLE)]
|
||||
|
||||
|
||||
def claim_next(conn: sqlite3.Connection) -> RemoteFile | None:
|
||||
"""Atomically take the oldest downloadable file and mark it in flight.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
RemoteFile | None: The claimed file, or None when nothing is pending.
|
||||
"""
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT md5, tim, ext, orig_name, fsize, width, height,
|
||||
thread_no, post_no
|
||||
FROM files
|
||||
WHERE state IN (?, ?, ?)
|
||||
ORDER BY first_seen, md5
|
||||
LIMIT 1
|
||||
""",
|
||||
claimable_values(),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
conn.execute("COMMIT")
|
||||
return None
|
||||
conn.execute(
|
||||
"UPDATE files SET state = ?, attempts = attempts + 1 WHERE md5 = ?",
|
||||
(FileState.DOWNLOADING.value, row["md5"]),
|
||||
)
|
||||
conn.execute("COMMIT")
|
||||
except Exception:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
return RemoteFile(
|
||||
md5=row["md5"],
|
||||
tim=row["tim"],
|
||||
ext=row["ext"],
|
||||
orig_name=row["orig_name"],
|
||||
fsize=row["fsize"],
|
||||
width=row["width"],
|
||||
height=row["height"],
|
||||
thread_no=row["thread_no"],
|
||||
post_no=row["post_no"],
|
||||
)
|
||||
|
||||
|
||||
def ready_items(conn: sqlite3.Connection, directory: Path) -> list[ReviewItem]:
|
||||
"""Return downloaded files still awaiting a verdict, oldest first.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
directory: Where downloaded files live.
|
||||
|
||||
Returns:
|
||||
list[ReviewItem]: Reviewable videos.
|
||||
"""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT md5, local_name, orig_name, ext, fsize, width, height
|
||||
FROM files
|
||||
WHERE state = ? AND local_name IS NOT NULL
|
||||
ORDER BY downloaded_at, md5
|
||||
""",
|
||||
(FileState.READY.value,),
|
||||
).fetchall()
|
||||
return [
|
||||
ReviewItem(
|
||||
md5=row["md5"],
|
||||
path=directory / row["local_name"],
|
||||
orig_name=f"{row['orig_name']}{row['ext']}",
|
||||
fsize=row["fsize"],
|
||||
width=row["width"],
|
||||
height=row["height"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def ready_item(source: RemoteFile, path: Path) -> ReviewItem:
|
||||
"""Build the reviewer's view of a file that just finished downloading.
|
||||
|
||||
Args:
|
||||
source: The catalogue record it came from.
|
||||
path: Where the verified bytes now live.
|
||||
|
||||
Returns:
|
||||
ReviewItem: Value handed to the UI thread.
|
||||
"""
|
||||
return ReviewItem(
|
||||
md5=source.md5,
|
||||
path=path,
|
||||
orig_name=f"{source.orig_name}{source.ext}",
|
||||
fsize=source.fsize,
|
||||
width=source.width,
|
||||
height=source.height,
|
||||
)
|
||||
|
||||
|
||||
def counts(conn: sqlite3.Connection) -> dict[str, int]:
|
||||
"""Return the number of files in each state.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
dict[str, int]: State value mapped to row count, plus ``total``.
|
||||
"""
|
||||
result = {
|
||||
str(row["state"]): int(row["n"])
|
||||
for row in conn.execute(
|
||||
"SELECT state, COUNT(*) AS n FROM files GROUP BY state",
|
||||
)
|
||||
}
|
||||
result["total"] = scalar(conn, "SELECT COUNT(*) FROM files")
|
||||
return result
|
||||
|
||||
|
||||
def pending_downloads(conn: sqlite3.Connection) -> int:
|
||||
"""Return how many files are still waiting to be fetched.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
int: Count of claimable rows.
|
||||
"""
|
||||
return scalar(
|
||||
conn,
|
||||
"SELECT COUNT(*) FROM files WHERE state IN (?, ?, ?)",
|
||||
claimable_values(),
|
||||
)
|
||||
|
||||
|
||||
def known_md5s(conn: sqlite3.Connection) -> set[str]:
|
||||
"""Return every md5 the index has ever seen.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
set[str]: Base64 md5 values, including terminal ones, so a reviewed or
|
||||
written-off file is never offered again.
|
||||
"""
|
||||
return {str(row[0]) for row in conn.execute("SELECT md5 FROM files")}
|
||||
|
||||
|
||||
def state_of(conn: sqlite3.Connection, md5_b64: str) -> FileState | None:
|
||||
"""Return a file's current state.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
md5_b64: File identity.
|
||||
|
||||
Returns:
|
||||
FileState | None: The state, or None when the file is unknown.
|
||||
"""
|
||||
row = conn.execute(
|
||||
"SELECT state FROM files WHERE md5 = ?",
|
||||
(md5_b64,),
|
||||
).fetchone()
|
||||
return None if row is None else FileState(row["state"])
|
||||
|
||||
|
||||
def attempts_for(conn: sqlite3.Connection, md5_b64: str) -> int:
|
||||
"""Return how many download attempts a file has had.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
md5_b64: File identity.
|
||||
|
||||
Returns:
|
||||
int: Attempt count, zero when the file is unknown.
|
||||
"""
|
||||
row = conn.execute(
|
||||
"SELECT attempts FROM files WHERE md5 = ?",
|
||||
(md5_b64,),
|
||||
).fetchone()
|
||||
return 0 if row is None else int(row["attempts"])
|
||||
@ -13,17 +13,49 @@ from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python_pkg.wsg_grabber.models import (
|
||||
RemoteFile,
|
||||
ReviewItem,
|
||||
# scalar() is shared: record_files counts rows before and after. The rest are
|
||||
# re-exported through __all__ so store.claim_next / store.ready_items / etc.
|
||||
# keep working for downloader.py, cli.py and test_store.py unchanged.
|
||||
from python_pkg.wsg_grabber._store_queries import (
|
||||
attempts_for,
|
||||
claim_next,
|
||||
claimable_values,
|
||||
counts,
|
||||
known_md5s,
|
||||
pending_downloads,
|
||||
ready_item,
|
||||
ready_items,
|
||||
scalar,
|
||||
state_of,
|
||||
)
|
||||
from python_pkg.wsg_grabber.states import CLAIMABLE, FileState
|
||||
from python_pkg.wsg_grabber.states import FileState
|
||||
|
||||
__all__ = [
|
||||
"attempts_for",
|
||||
"claim_next",
|
||||
"claimable_values",
|
||||
"counts",
|
||||
"known_md5s",
|
||||
"local_name",
|
||||
"mark_downloaded",
|
||||
"pending_downloads",
|
||||
"ready_item",
|
||||
"ready_items",
|
||||
"record_files",
|
||||
"record_progress",
|
||||
"reset_in_flight",
|
||||
"set_state",
|
||||
"state_of",
|
||||
]
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from collections.abc import Iterable
|
||||
import sqlite3
|
||||
|
||||
from python_pkg.wsg_grabber.models import (
|
||||
RemoteFile,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
"""Return the current UTC time as an ISO-8601 string.
|
||||
@ -84,7 +116,7 @@ def record_files(conn: sqlite3.Connection, files: Iterable[RemoteFile]) -> int:
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
before = _scalar(conn, "SELECT COUNT(*) FROM files")
|
||||
before = scalar(conn, "SELECT COUNT(*) FROM files")
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO files (md5, tim, ext, orig_name, fsize, width, height,
|
||||
@ -94,113 +126,7 @@ def record_files(conn: sqlite3.Connection, files: Iterable[RemoteFile]) -> int:
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
return _scalar(conn, "SELECT COUNT(*) FROM files") - before
|
||||
|
||||
|
||||
def known_md5s(conn: sqlite3.Connection) -> set[str]:
|
||||
"""Return every md5 the index has ever seen.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
set[str]: Base64 md5 values, including terminal ones, so a reviewed or
|
||||
written-off file is never offered again.
|
||||
"""
|
||||
return {str(row[0]) for row in conn.execute("SELECT md5 FROM files")}
|
||||
|
||||
|
||||
def claimable_values() -> list[str]:
|
||||
"""Return the state values a download worker may pick up, in a stable order.
|
||||
|
||||
The queries below spell out one placeholder per member rather than building
|
||||
the ``IN`` list at runtime, so the SQL stays a literal. ``test_store``
|
||||
pins the member count so adding a state cannot silently desync the two.
|
||||
|
||||
Returns:
|
||||
list[str]: Sorted state values.
|
||||
"""
|
||||
return [state.value for state in sorted(CLAIMABLE)]
|
||||
|
||||
|
||||
def claim_next(conn: sqlite3.Connection) -> RemoteFile | None:
|
||||
"""Atomically take the oldest downloadable file and mark it in flight.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
RemoteFile | None: The claimed file, or None when nothing is pending.
|
||||
"""
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT md5, tim, ext, orig_name, fsize, width, height,
|
||||
thread_no, post_no
|
||||
FROM files
|
||||
WHERE state IN (?, ?, ?)
|
||||
ORDER BY first_seen, md5
|
||||
LIMIT 1
|
||||
""",
|
||||
claimable_values(),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
conn.execute("COMMIT")
|
||||
return None
|
||||
conn.execute(
|
||||
"UPDATE files SET state = ?, attempts = attempts + 1 WHERE md5 = ?",
|
||||
(FileState.DOWNLOADING.value, row["md5"]),
|
||||
)
|
||||
conn.execute("COMMIT")
|
||||
except Exception:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
return RemoteFile(
|
||||
md5=row["md5"],
|
||||
tim=row["tim"],
|
||||
ext=row["ext"],
|
||||
orig_name=row["orig_name"],
|
||||
fsize=row["fsize"],
|
||||
width=row["width"],
|
||||
height=row["height"],
|
||||
thread_no=row["thread_no"],
|
||||
post_no=row["post_no"],
|
||||
)
|
||||
|
||||
|
||||
def state_of(conn: sqlite3.Connection, md5_b64: str) -> FileState | None:
|
||||
"""Return a file's current state.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
md5_b64: File identity.
|
||||
|
||||
Returns:
|
||||
FileState | None: The state, or None when the file is unknown.
|
||||
"""
|
||||
row = conn.execute(
|
||||
"SELECT state FROM files WHERE md5 = ?",
|
||||
(md5_b64,),
|
||||
).fetchone()
|
||||
return None if row is None else FileState(row["state"])
|
||||
|
||||
|
||||
def attempts_for(conn: sqlite3.Connection, md5_b64: str) -> int:
|
||||
"""Return how many download attempts a file has had.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
md5_b64: File identity.
|
||||
|
||||
Returns:
|
||||
int: Attempt count, zero when the file is unknown.
|
||||
"""
|
||||
row = conn.execute(
|
||||
"SELECT attempts FROM files WHERE md5 = ?",
|
||||
(md5_b64,),
|
||||
).fetchone()
|
||||
return 0 if row is None else int(row["attempts"])
|
||||
return scalar(conn, "SELECT COUNT(*) FROM files") - before
|
||||
|
||||
|
||||
def set_state(
|
||||
@ -257,58 +183,6 @@ def record_progress(conn: sqlite3.Connection, md5_b64: str, done: int) -> None:
|
||||
)
|
||||
|
||||
|
||||
def ready_items(conn: sqlite3.Connection, directory: Path) -> list[ReviewItem]:
|
||||
"""Return downloaded files still awaiting a verdict, oldest first.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
directory: Where downloaded files live.
|
||||
|
||||
Returns:
|
||||
list[ReviewItem]: Reviewable videos.
|
||||
"""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT md5, local_name, orig_name, ext, fsize, width, height
|
||||
FROM files
|
||||
WHERE state = ? AND local_name IS NOT NULL
|
||||
ORDER BY downloaded_at, md5
|
||||
""",
|
||||
(FileState.READY.value,),
|
||||
).fetchall()
|
||||
return [
|
||||
ReviewItem(
|
||||
md5=row["md5"],
|
||||
path=directory / row["local_name"],
|
||||
orig_name=f"{row['orig_name']}{row['ext']}",
|
||||
fsize=row["fsize"],
|
||||
width=row["width"],
|
||||
height=row["height"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def ready_item(source: RemoteFile, path: Path) -> ReviewItem:
|
||||
"""Build the reviewer's view of a file that just finished downloading.
|
||||
|
||||
Args:
|
||||
source: The catalogue record it came from.
|
||||
path: Where the verified bytes now live.
|
||||
|
||||
Returns:
|
||||
ReviewItem: Value handed to the UI thread.
|
||||
"""
|
||||
return ReviewItem(
|
||||
md5=source.md5,
|
||||
path=path,
|
||||
orig_name=f"{source.orig_name}{source.ext}",
|
||||
fsize=source.fsize,
|
||||
width=source.width,
|
||||
height=source.height,
|
||||
)
|
||||
|
||||
|
||||
def reset_in_flight(conn: sqlite3.Connection) -> int:
|
||||
"""Return files interrupted mid-download to the pending pool.
|
||||
|
||||
@ -326,57 +200,3 @@ def reset_in_flight(conn: sqlite3.Connection) -> int:
|
||||
(FileState.FAILED.value, FileState.DOWNLOADING.value),
|
||||
)
|
||||
return int(cursor.rowcount)
|
||||
|
||||
|
||||
def counts(conn: sqlite3.Connection) -> dict[str, int]:
|
||||
"""Return the number of files in each state.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
dict[str, int]: State value mapped to row count, plus ``total``.
|
||||
"""
|
||||
result = {
|
||||
str(row["state"]): int(row["n"])
|
||||
for row in conn.execute(
|
||||
"SELECT state, COUNT(*) AS n FROM files GROUP BY state",
|
||||
)
|
||||
}
|
||||
result["total"] = _scalar(conn, "SELECT COUNT(*) FROM files")
|
||||
return result
|
||||
|
||||
|
||||
def pending_downloads(conn: sqlite3.Connection) -> int:
|
||||
"""Return how many files are still waiting to be fetched.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
|
||||
Returns:
|
||||
int: Count of claimable rows.
|
||||
"""
|
||||
return _scalar(
|
||||
conn,
|
||||
"SELECT COUNT(*) FROM files WHERE state IN (?, ?, ?)",
|
||||
claimable_values(),
|
||||
)
|
||||
|
||||
|
||||
def _scalar(
|
||||
conn: sqlite3.Connection,
|
||||
sql: str,
|
||||
params: Sequence[object] | Mapping[str, object] | None = None,
|
||||
) -> int:
|
||||
"""Run a single-value query and return it as an int.
|
||||
|
||||
Args:
|
||||
conn: Open connection.
|
||||
sql: Query yielding exactly one column.
|
||||
params: Optional bound parameters.
|
||||
|
||||
Returns:
|
||||
int: The first column of the first row.
|
||||
"""
|
||||
cursor = conn.execute(sql) if params is None else conn.execute(sql, params)
|
||||
return int(cursor.fetchone()[0])
|
||||
|
||||
Loading…
Reference in New Issue
Block a user