#!/usr/bin/env python3 # Copyright (c) 2026 Krzysztof Rudnicki. MIT licence. """Where an interrupted harvest leaves the work it already did. Its own module because it is its own concern: neither talking to Commons (commons_api) nor deciding which files belong in a pool (commons_harvest), but making the second survivable. A harvest is an hour of somebody else's bandwidth, and killing one used to throw all of it away -- three times in a single afternoon, which is what prompted this. The unit is the category. Whatever finished stays finished, so a second run pays only for what the first did not reach, and a completed harvest re-runs in seconds. """ from __future__ import annotations import json from typing import TYPE_CHECKING if TYPE_CHECKING: from pathlib import Path def load_cache(path: Path) -> dict[str, dict[int, str]]: """Categories already walked, from a previous run of this generator. A harvest is an hour of somebody else's bandwidth, and killing it used to throw all of that away -- three times in one afternoon, which is what prompted this. The unit is the category rather than the run: whatever was finished stays finished, so a second run pays only for what the first did not reach, and a completed harvest re-runs in seconds. JSON object keys are strings, so the page ids come back as strings and are converted here rather than at every use. """ if not path.is_file(): return {} try: document = json.loads(path.read_text(encoding="utf-8")) except OSError, json.JSONDecodeError: # A half-written cache is a reason to redo the work, never a reason # to fail: the whole point of this file is that it is disposable. return {} return { category: {int(page): str(title) for page, title in files.items()} for category, files in document.items() } def save_cache(path: Path, cache: dict[str, dict[int, str]]) -> None: """Write the cache, atomically, so an interrupted save cannot poison it.""" temporary = path.with_suffix(".tmp") temporary.write_text(json.dumps(cache), encoding="utf-8") temporary.replace(path)