diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 4f7d4940..6ce5aa66 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -7,7 +7,6 @@ on: - "python_pkg/lichess_bot/**" - "python_pkg/**" - "tests/**" - - "linux_configuration/scripts/periodic_background/system-maintenance/bin/**" - "linux_configuration/tests/**" # Both spellings: the root requirements.txt is a SYMLINK into meta/, # and GitHub matches path filters against the changed path as git @@ -24,7 +23,6 @@ on: - "python_pkg/lichess_bot/**" - "python_pkg/**" - "tests/**" - - "linux_configuration/scripts/periodic_background/system-maintenance/bin/**" - "linux_configuration/tests/**" # Both spellings: the root requirements.txt is a SYMLINK into meta/, # and GitHub matches path filters against the changed path as git diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 8bbbabe9..f960def0 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -139,7 +139,6 @@ jobs: run: | set -euo pipefail tests=( - test_hosts_file_monitor.sh test_hosts_guard_pacman_integration.sh test_i3blocks_efficiency.sh test_makepkg_capped.sh diff --git a/AGENTS.md b/AGENTS.md index c30cc09e..2896b259 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,9 @@ Extracted to their own repos: [`steam-backlog-enforcer`](https://github.com/kuhy [`reverse-survivors`](https://github.com/kuhyx/reverse-survivors), [`kcd2-dice-solver`](https://github.com/kuhyx/kcd2-dice-solver) (lives at `~/kcd2-dice-solver`), [`focus-owner`](https://github.com/kuhyx/focus-owner), -[`mtk-root`](https://github.com/kuhyx/mtk-root). +[`mtk-root`](https://github.com/kuhyx/mtk-root), +[`system-maintenance`](https://github.com/kuhyx/system-maintenance) (lives at +`~/system-maintenance`). Archived work: [`testsAndMisc-archive`](https://github.com/kuhyx/testsAndMisc-archive). ## Adding a Flutter app diff --git a/docs/superpowers/contracts/extract-system-maintenance-2026-08-23.json b/docs/superpowers/contracts/extract-system-maintenance-2026-08-23.json new file mode 100644 index 00000000..ce810102 --- /dev/null +++ b/docs/superpowers/contracts/extract-system-maintenance-2026-08-23.json @@ -0,0 +1,20 @@ +{ + "title": "Extract system-maintenance, the first of the four live subsystems", + "objective": "Move the periodic maintenance subsystem to its own public repo. It is the isolated one -- no cross-unit source edges -- so it goes first and establishes the pattern for the three harder ones. Its Python tests live in linux_configuration/tests/ and reach it through a shared conftest, so the test half must move with it without breaking the suites that stay.", + "acceptance_criteria": [ + "A vmbox pass first: the installer runs from an extracted-layout copy in the guest before anything is changed on the host, and the result is reported as sandbox-only.", + "The new repo's Python tests pass from a clean clone with its own conftest; 136 tests.", + "Both extracted shell suites pass after their hardcoded monorepo paths are fixed, and shellcheck exits 0 with no severity filter.", + "The new repo declares BOTH CI halves, since python-tests.yml reached it by paths filter and shell-tests.yml by name list, and neither follows the code out.", + "The suites remaining in the monorepo still pass: 202 Python tests.", + "The cross-unit pair test test_shutdown_timer_monitor.sh keeps working, and does not silently degrade into asserting against a stale installed binary.", + "No workflow, conftest, pyproject sys.path entry or coverage-allowlist line names a removed path.", + "pre-commit passes without --no-verify." + ], + "out_of_scope": [ + "Refreshing the stale /usr/local/bin/shutdown-timer-monitor.sh, which digital_wellbeing's installer owns.", + "The other three subsystems.", + "Moving the shared linux_configuration/tests/conftest.py, which serves several script directories that stay." + ], + "verifier": "vm run in guest exercises the installer; python -m pytest tests -q reports 136 passed in the new repo; bash tests/*.sh and shellcheck both clean there; pytest linux_configuration/tests reports 202 passed here; bash linux_configuration/tests/test_shutdown_timer_monitor.sh passes with the checkout and SKIPs without it; check_shell_coverage.sh --all exits 0; check_directory_depth.sh --all drops 294 -> 267." +} diff --git a/docs/superpowers/evidence/extract-system-maintenance-2026-08-23.json b/docs/superpowers/evidence/extract-system-maintenance-2026-08-23.json new file mode 100644 index 00000000..2fac00c6 --- /dev/null +++ b/docs/superpowers/evidence/extract-system-maintenance-2026-08-23.json @@ -0,0 +1,59 @@ +{ + "intent": "Extract the periodic system-maintenance subsystem into its own public repo with the 21 test files that lived in linux_configuration/tests/, and keep the one cross-unit pair test working here.", + "scope": [ + "linux_configuration/scripts/periodic_background/system-maintenance/ (32 files) -> github.com/kuhyx/system-maintenance, cloned to ~/system-maintenance", + "20 of its 21 test files in linux_configuration/tests/ moved with it", + "test_shutdown_timer_monitor.sh KEPT here: it is the pair test against digital_wellbeing's installer", + "conftest.py, meta/pyproject.toml, python-tests.yml, shell-tests.yml, shell-coverage-allowlist.txt cleaned", + "Non-goal: the other three subsystems" + ], + "changes": [ + "Extracted the subsystem and its tests in one filter-repo pass (28 commits), hoisting bin/, systemd/, logrotate/ to the repo root and linux_configuration/tests/ to tests/.", + "Gave the new repo its own tests/conftest.py putting bin/ on sys.path, plus a pyproject.toml with testpaths and branch coverage. The monorepo's shared conftest served several script directories and stays.", + "Fixed two extracted shell suites that hardcoded $REPO_DIR/scripts/periodic_background/system-maintenance/bin, now $REPO_DIR/bin.", + "Re-declared both CI halves in the new repo: python-tests.yml reached these modules through a paths filter and shell-tests.yml through a name list, neither of which follows the code out.", + "Repointed the kept pair test at ${SYSTEM_MAINTENANCE_DIR:-$HOME/system-maintenance}, skipping cleanly when absent.", + "Removed test_hosts_file_monitor.sh from shell-tests.yml's explicit list, since it left with the subsystem and CI would otherwise fail on a missing file." + ], + "verification": [ + { + "command": "sandbox: vm new sm1 + run the installer from an extracted-layout copy", + "result": "pass in vmbox, NOT host-verified", + "evidence": "install_usage_monitoring.sh detected the Arch distro family and resolved its 35 packages from a copy rooted outside the monorepo. A grep for testsAndMisc/periodic_background/linux_configuration across the extracted tree found only one stale comment, confirming no path coupling." + }, + { + "command": "python -m pytest tests -q (extracted repo)", + "result": "pass", + "evidence": "136 passed." + }, + { + "command": "bash tests/*.sh (extracted repo) + shellcheck", + "result": "pass", + "evidence": "Both remaining shell suites pass after the path fix; shellcheck exits 0 over every script with .shellcheckrc carried across." + }, + { + "command": "pytest linux_configuration/tests (monorepo, after removal)", + "result": "pass", + "evidence": "202 passed." + }, + { + "command": "bash linux_configuration/tests/test_shutdown_timer_monitor.sh", + "result": "pass", + "evidence": "Passes against the ~/system-maintenance checkout, matching the baseline worktree's behaviour. An intermediate version fell back to /usr/local/bin and HUNG: the installed copy is stale and differs from source, so the fallback was removed rather than kept. SYSTEM_MAINTENANCE_DIR=/nonexistent prints SKIP and exits 0." + }, + { + "command": "bash meta/scripts/check_shell_coverage.sh --all", + "result": "pass", + "evidence": "Exit 0 with the five system-maintenance allowlist entries removed." + } + ], + "risks": [ + "/usr/local/bin/shutdown-timer-monitor.sh is stale relative to source. Left alone here; digital_wellbeing's installer owns reinstalling it, and that is the next extraction.", + "The pair test now depends on a sibling checkout at ~/system-maintenance. It skips rather than fails when absent, so CI without the checkout reports a skip, not a false pass on assertions it never ran.", + "/usr/local/bin/periodic-system-maintenance.sh still names digital_wellbeing and hosts paths inside this repo; those extractions must reinstall it." + ], + "rollback": [ + "git revert to restore the subsystem and its 21 tests, then re-add the conftest entry, pyproject sys.path element and both workflow references.", + "Re-run pytest linux_configuration/tests and check_shell_coverage.sh --all to confirm the restored state." + ] +} diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/__init__.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/__init__.py deleted file mode 100644 index ca28cd93..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for usage_report: parsing and type definitions.""" diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_atop.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_atop.py deleted file mode 100644 index 7886ce6d..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_atop.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Subprocess plumbing for usage_report. - -Runs atop, streams its parseable output, and builds/caches the native -`atop_agg` C helper. -""" - -from __future__ import annotations - -import datetime as _dt -from pathlib import Path -import shutil -import subprocess -from typing import TYPE_CHECKING - -from _usage_report_types import ProcAgg, _Window - -if TYPE_CHECKING: - from collections.abc import Iterator - - from _usage_report_types import _Progress - -_ATOP_AGG_CACHE_BIN = Path.home() / ".cache" / "usage_report" / "atop_agg" -_ATOP_AGG_BIN_MODE = 0o755 -# Repo layout: linux_configuration/scripts/system-maintenance/bin/usage_report.py -# -> parents[4] is the repo root which hosts the C/ source tree. -_ATOP_AGG_SRC_DIR = Path(__file__).resolve().parents[4] / "C" / "atop_agg" -_ATOP_AGG_BUILD_TIMEOUT_S = 60 -_NATIVE_TSV_NAME_LEN = 7 -_NATIVE_TSV_WIN_LEN = 5 - - -def _run(cmd: list[str]) -> str: - """Run *cmd* and return stdout (empty string on failure).""" - try: - proc = subprocess.run( - cmd, - capture_output=True, - text=True, - check=False, - timeout=60, - ) - except (OSError, subprocess.TimeoutExpired): - return "" - return proc.stdout - - -def _atop_read_cmd( - log: Path, - labels: str, - begin: str | None, - end: str | None, -) -> list[str]: - """Build an `atop -r` command, optionally bounded by begin/end times. - - *begin*/*end* are atop `-b`/`-e` arguments (`[YYYYMMDD]hhmm[ss]`) used to - restrict replay to a sub-window of the day's log, so a "since last report" - run does not double-count the part of the first day already reported. - """ - cmd = ["atop", "-r", str(log)] - if begin is not None: - cmd += ["-b", begin] - if end is not None: - cmd += ["-e", end] - cmd += ["-P", labels] - return cmd - - -def _iter_atop_lines( - log: Path, - labels: str, - begin: str | None = None, - end: str | None = None, -) -> Iterator[str]: - """Stream `atop -r LOG -P LABELS` stdout line-by-line. - - Uses `Popen` so the report can show progress while atop is still - decoding its binary log, rather than buffering the whole output. - """ - cmd = _atop_read_cmd(log, labels, begin, end) - with subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) as proc: - stdout = proc.stdout - if stdout is None: - return - for raw in stdout: - yield raw.rstrip("\n") - - -def _atop_agg_binary() -> Path | None: - """Return a cached `atop_agg` binary path, auto-building if missing/stale. - - Falls back to ``None`` when the C source tree or a system C compiler - is unavailable, in which case callers use the pure-Python parser. - """ - src_c = _ATOP_AGG_SRC_DIR / "atop_agg.c" - if not src_c.exists(): - # Source tree is gone (relocated/extracted): never trust an orphaned - # cached binary whose provenance we can no longer verify against - # source — a stale build can silently carry parsing bugs. Fall back to - # the pure-Python parser instead. - return None - if ( - _ATOP_AGG_CACHE_BIN.exists() - and src_c.stat().st_mtime <= _ATOP_AGG_CACHE_BIN.stat().st_mtime - ): - return _ATOP_AGG_CACHE_BIN - if shutil.which("cc") is None: - return None - _ATOP_AGG_CACHE_BIN.parent.mkdir(parents=True, exist_ok=True) - make_cmd = ["make", "-s", "-C", str(_ATOP_AGG_SRC_DIR), "atop_agg"] - try: - subprocess.run( - make_cmd, - check=True, - capture_output=True, - timeout=_ATOP_AGG_BUILD_TIMEOUT_S, - ) - except (OSError, subprocess.SubprocessError): - return None - built = _ATOP_AGG_SRC_DIR / "atop_agg" - if not built.exists(): - return None - shutil.copy2(built, _ATOP_AGG_CACHE_BIN) - _ATOP_AGG_CACHE_BIN.chmod(_ATOP_AGG_BIN_MODE) - return _ATOP_AGG_CACHE_BIN - - -def _apply_native_name(parts: list[str], agg_map: dict[str, ProcAgg]) -> None: - r"""Fold one `N\t\t\t\t\t\t` row.""" - _, name, cpu_s, peak_s, sum_avg_s, rss_n_s, pids_s = parts - entry = agg_map.setdefault(name, ProcAgg(name=name)) - entry.cpu_ticks = int(cpu_s) - entry.peak_rss_kb = int(peak_s) - entry.rss_kb_sum = int(sum_avg_s) - entry.rss_samples = int(rss_n_s) - # The C helper pre-aggregates by name; pid_set is unused in the native - # path but `len(pid_set)` drives the "PIDs" column in the report. - entry.pid_set = set(range(int(pids_s))) - - -def _window_from_native(parts: list[str]) -> _Window: - r"""Build a `_Window` from a `W\t\t\t\t` row.""" - _, start_s, end_s, n_s, interval_s = parts - n_epochs = int(n_s) - if not n_epochs: - return _Window() - start_epoch = int(start_s) - end_epoch = int(end_s) - start_dt = _dt.datetime.fromtimestamp(start_epoch).astimezone() - end_dt = _dt.datetime.fromtimestamp(end_epoch).astimezone() - return _Window( - start=start_dt.isoformat(timespec="seconds"), - end=end_dt.isoformat(timespec="seconds"), - distinct_samples=n_epochs, - interval_s=int(interval_s), - seconds=end_epoch - start_epoch, - start_epoch=start_epoch, - end_epoch=end_epoch, - ) - - -def _aggregate_atop_native( - log: Path, - progress: _Progress, - binary: Path, - begin: str | None = None, - end: str | None = None, -) -> tuple[dict[str, ProcAgg], _Window]: - """Aggregate via `atop | atop_agg`; return `(by_name, window)`.""" - progress.start_stage("atop: parse PRC+PRM (native)") - agg_map: dict[str, ProcAgg] = {} - window = _Window() - atop_cmd = _atop_read_cmd(log, "PRC,PRM", begin, end) - agg_cmd = [str(binary)] - with ( - subprocess.Popen( - atop_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) as atop, - subprocess.Popen( - agg_cmd, - stdin=atop.stdout, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) as agg, - ): - if atop.stdout is not None: - atop.stdout.close() - stdout = agg.stdout - if stdout is None: - return agg_map, window - for raw in stdout: - parts = raw.rstrip("\n").split("\t") - tag = parts[0] - if tag == "N" and len(parts) == _NATIVE_TSV_NAME_LEN: - _apply_native_name(parts, agg_map) - elif tag == "W" and len(parts) == _NATIVE_TSV_WIN_LEN: - window = _window_from_native(parts) - progress.update(1.0) - return agg_map, window diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_emit.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_emit.py deleted file mode 100644 index e973fcc8..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_emit.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Preflight checks and report output for usage_report. - -Verifies atop and its log are present before a run, writes the finished report -to stdout, and copies it to the system clipboard with whichever tool is -installed. -""" - -from __future__ import annotations - -from pathlib import Path -import shutil -import subprocess -import sys -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - import argparse - - -_INSTALL_SCRIPT = Path(__file__).with_name("install_usage_monitoring.sh") - - -def _preflight(atop_log: Path) -> None: - if not shutil.which("atop"): - sys.exit( - f"error: `atop` is not installed.\nrun: {_INSTALL_SCRIPT}", - ) - if not atop_log.exists(): - sys.exit( - f"error: atop log not found: {atop_log}\n" - f"run: {_INSTALL_SCRIPT} (enables atop.service), " - "then wait for the first sample.", - ) - - -_CLIPBOARD_CANDIDATES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("wl-copy", ("wl-copy",)), - ("xclip", ("xclip", "-selection", "clipboard")), - ("xsel", ("xsel", "--clipboard", "--input")), -) - - -def _copy_to_clipboard(text: str) -> None: - """Copy `text` to the system clipboard using the first available tool. - - Prints a one-line status to stderr so the stdout report stays pristine - for redirection. - """ - for name, cmd in _CLIPBOARD_CANDIDATES: - if not shutil.which(name): - continue - try: - subprocess.run(cmd, input=text, text=True, check=True) - except (subprocess.CalledProcessError, OSError) as exc: - sys.stderr.write(f"clipboard: {name} failed: {exc}\n") - return - sys.stderr.write(f"clipboard: copied {len(text)} chars via {name}\n") - return - sys.stderr.write( - "clipboard: no wl-copy/xclip/xsel found; skipping copy\n", - ) - - -def _emit(args: argparse.Namespace, report: str) -> None: - """Write the report to stdout and (unless suppressed) the clipboard.""" - sys.stdout.write(report) - if not args.no_clipboard: - _copy_to_clipboard(report) diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_format.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_format.py deleted file mode 100644 index 87698a4c..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_format.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Markdown escaping and duration formatting shared by the report sections.""" - -from __future__ import annotations - -_SEC_PER_HOUR = 3600 -_SEC_PER_MIN = 60 - - -def _md_escape(name: str) -> str: - """Escape characters that would break a Markdown table cell.""" - return name.replace("|", r"\|").replace("\n", " ") - - -def _fmt_h(seconds: float) -> str: - """Human-friendly duration: `"1h 23m"` / `"4m 12s"` / `"8.3s"`.""" - if seconds >= _SEC_PER_HOUR: - h = int(seconds // _SEC_PER_HOUR) - m = int((seconds % _SEC_PER_HOUR) // _SEC_PER_MIN) - return f"{h}h {m:02d}m" - if seconds >= _SEC_PER_MIN: - m = int(seconds // _SEC_PER_MIN) - s = int(seconds % _SEC_PER_MIN) - return f"{m}m {s:02d}s" - return f"{seconds:.1f}s" diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_logs.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_logs.py deleted file mode 100644 index 205aac5b..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_logs.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Log discovery, report state and per-day segment planning for usage_report. - -Resolves which daily atop/pmon logs a reporting window covers, persists the -"last report generated" timestamp that drives the since-last-report mode, and -describes the logs actually consumed for the report's methodology section. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import datetime as _dt -import json -from pathlib import Path - -from _usage_report_types import _PMON_INTERVAL_S - -_ATOP_LOG_DIR = Path("/var/log/atop") -_PMON_LOG_DIR = Path.home() / ".local/share/gpu-log" -# Persisted marker of when the last report was generated. Lives under -# ~/.local/share (durable app state), not ~/.cache, so clearing caches does not -# silently reset the "since last report" window back to today-only. -_STATE_DIR = Path.home() / ".local/share/usage_report" -_STATE_FILE = _STATE_DIR / "last_report.json" - - -@dataclass -class _Segment: - """One calendar day's resolved logs plus optional in-day start bounds. - - *atop_begin* is an atop ``-b`` argument (``YYYYMMDDhhmmss``) and - *pmon_begin_epoch* the matching local epoch; both are set only for the first - day of a "since last report" window so re-runs do not double-count. - """ - - atop_log: Path - pmon_log: Path - atop_begin: str | None = None - pmon_begin_epoch: float | None = None - - -def _resolve_logs(date: str) -> tuple[Path, Path]: - atop_log = _ATOP_LOG_DIR / f"atop_{date}" - pmon_log = _PMON_LOG_DIR / f"pmon-{date}.log" - return atop_log, pmon_log - - -def _read_last_generated() -> _dt.datetime | None: - """Return the timestamp of the previous report run, or None if unknown.""" - try: - raw = _STATE_FILE.read_text(encoding="utf-8") - except OSError: - return None - try: - stamp = json.loads(raw)["last_generated"] - return _dt.datetime.fromisoformat(stamp).astimezone() - except (ValueError, KeyError, TypeError): - return None - - -def _write_last_generated(when: _dt.datetime) -> None: - """Persist *when* as the last-report timestamp for the next run.""" - _STATE_DIR.mkdir(parents=True, exist_ok=True) - payload = json.dumps({"last_generated": when.isoformat(timespec="seconds")}) - _STATE_FILE.write_text(payload + "\n", encoding="utf-8") - - -def _has_time_of_day(when: _dt.datetime) -> bool: - """True when *when* is past local midnight, so a begin bound is needed.""" - return bool(when.hour or when.minute or when.second or when.microsecond) - - -def _plan_segments(start: _dt.datetime, end: _dt.datetime) -> list[_Segment]: - """Resolve one `_Segment` per calendar day across ``[start, end]``. - - The first day is bounded at *start*'s time-of-day so a same-day re-run only - covers the slice since the previous report; later days are covered in full. - Returns an empty list when *start* is after *end* (e.g. a future state file). - """ - segments: list[_Segment] = [] - day = start.date() - while day <= end.date(): - atop_log, pmon_log = _resolve_logs(day.strftime("%Y%m%d")) - if day == start.date() and _has_time_of_day(start): - segments.append( - _Segment( - atop_log, - pmon_log, - start.strftime("%Y%m%d%H%M%S"), - start.timestamp(), - ), - ) - else: - segments.append(_Segment(atop_log, pmon_log)) - day += _dt.timedelta(days=1) - return segments - - -def _describe_logs(paths: list[Path], how: str) -> str: - """One-line Markdown description of the log files actually consumed.""" - if not paths: - return f"_none found_ (`{how}`)" - if len(paths) == 1: - return f"`{paths[0]}` (`{how}`)" - return ( - f"{len(paths)} daily logs `{paths[0].name}` … `{paths[-1].name}` " - f"in `{paths[0].parent}` (`{how}`)" - ) - - -def _log_descriptions(segments: list[_Segment]) -> tuple[str, str]: - """Return ``(atop_desc, pmon_desc)`` for the logs present in *segments*.""" - atop_present = [seg.atop_log for seg in segments if seg.atop_log.exists()] - pmon_present = [seg.pmon_log for seg in segments if seg.pmon_log.exists()] - return ( - _describe_logs(atop_present, "atop -r"), - _describe_logs(pmon_present, f"nvidia-smi pmon -d {_PMON_INTERVAL_S}"), - ) diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_parsing.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_parsing.py deleted file mode 100644 index 48ccd247..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_parsing.py +++ /dev/null @@ -1,243 +0,0 @@ -"""atop record parsing and per-day aggregate merging for usage_report.""" - -from __future__ import annotations - -import contextlib -import datetime as _dt -from typing import TYPE_CHECKING - -from _usage_report_atop import ( - _aggregate_atop_native, - _atop_agg_binary, - _iter_atop_lines, -) -from _usage_report_types import ( - _MIN_SAMPLES_FOR_WINDOW, - GpuAgg, - ProcAgg, - _PidCpu, - _PidRam, - _Window, -) - -if TYPE_CHECKING: - from pathlib import Path - - from _usage_report_types import _Progress - -# atop parseable output layout (atop 2.x, same on Arch/Debian/Ubuntu): -# 0 label, 1 host, 2 epoch, 3 YYYY/MM/DD, 4 HH:MM:SS, 5 interval_s, -# then per-process fields starting at index 6. -# PRC per-proc: pid name(parens) state HZ utime_ticks stime_ticks ... -# NOTE: atop inserts its clock-tick rate (HZ) between `state` and `utime` -# (the PRC analogue of the pagesize field PRM inserts before its memory -# columns); utime/stime therefore live two and three slots past `state`. -_PRC_PID_IDX = 6 -_PRC_NAME_IDX = 7 -_PRC_MIN_LEN = 12 -# PRM per-proc: pid name state pagesz_b vsize_kb rsize_kb ... -_PRM_PID_IDX = 6 -_PRM_NAME_IDX = 7 -_PRM_MIN_LEN = 12 -_CPU_RECORD_MIN_LEN = 5 -_PAREN_PAIR_MIN = 2 - - -def _parse_name(parts: list[str], name_idx: int) -> tuple[str, int]: - """Extract `(name, next_index)` from atop parseable output. - - atop wraps process names in parentheses and the name itself may contain - spaces, so we rejoin until we hit the closing `)`. Fast-paths the common - case where the name is a single token (no embedded spaces). - """ - if name_idx >= len(parts): - return "unknown", name_idx + 1 - token = parts[name_idx] - # Fast path: `(name)` fully in one token. - if len(token) >= _PAREN_PAIR_MIN and token[0] == "(" and token[-1] == ")": - return token[1:-1] or "unknown", name_idx + 1 - if token.startswith("("): - buf = [token] - idx = name_idx - while not buf[-1].endswith(")") and idx + 1 < len(parts): - idx += 1 - buf.append(parts[idx]) - name = " ".join(buf)[1:-1] or "unknown" - return name, idx + 1 - return token, name_idx + 1 - - -def _parse_prc(parts: list[str], pid_cpu: dict[int, _PidCpu]) -> None: - """Fold one PRC record into the per-PID CPU-ticks map.""" - try: - pid = int(parts[_PRC_PID_IDX]) - except (ValueError, IndexError): - return - name, after = _parse_name(parts, _PRC_NAME_IDX) - # After name comes: state HZ utime stime ... (HZ is atop's clock-tick - # rate; skipping it is what keeps a constant 100 from being charged as - # CPU to every record — the bug that made cpu-seconds collapse to PID - # count for short-lived processes). - try: - utime = int(parts[after + 2]) - stime = int(parts[after + 3]) - except (ValueError, IndexError): - return - pid_cpu.setdefault(pid, _PidCpu()).observe(name, utime + stime) - - -def _parse_prm(parts: list[str], pid_ram: dict[int, _PidRam]) -> None: - """Fold one PRM record into the per-PID RSS map.""" - try: - pid = int(parts[_PRM_PID_IDX]) - except (ValueError, IndexError): - return - name, after = _parse_name(parts, _PRM_NAME_IDX) - # After name: state pagesz_b vsize_kb rsize_kb ... - try: - rsize_kb = int(parts[after + 3]) - except (ValueError, IndexError): - return - pid_ram.setdefault(pid, _PidRam()).observe(name, rsize_kb) - - -def _window_from_epochs(epochs: set[int]) -> _Window: - """Build a `_Window` from a set of sample epoch timestamps.""" - if not epochs: - return _Window() - ordered = sorted(epochs) - start_dt = _dt.datetime.fromtimestamp(ordered[0]).astimezone() - end_dt = _dt.datetime.fromtimestamp(ordered[-1]).astimezone() - interval = 0 - if len(ordered) >= _MIN_SAMPLES_FOR_WINDOW: - deltas = sorted(ordered[i + 1] - ordered[i] for i in range(len(ordered) - 1)) - interval = deltas[len(deltas) // 2] - return _Window( - start=start_dt.isoformat(timespec="seconds"), - end=end_dt.isoformat(timespec="seconds"), - distinct_samples=len(ordered), - interval_s=interval, - seconds=ordered[-1] - ordered[0], - start_epoch=ordered[0], - end_epoch=ordered[-1], - ) - - -def aggregate_atop( - log: Path, - progress: _Progress, - begin: str | None = None, - end: str | None = None, -) -> tuple[dict[str, ProcAgg], _Window]: - """Stream PRC+PRM records, fold them into `{name: ProcAgg}`, return window. - - Prefers the native `atop_agg` C helper (auto-built into - ``~/.cache/usage_report/``) for ~7x speedup on full-day logs, falling - back to an inline Python parser when the helper is unavailable. - - *begin*/*end* are optional atop `-b`/`-e` arguments that bound replay to a - sub-window of the day's log (used by the "since last report" mode). - """ - binary = _atop_agg_binary() - if binary is not None: - return _aggregate_atop_native(log, progress, binary, begin, end) - progress.start_stage("atop: parse PRC+PRM") - pid_cpu: dict[int, _PidCpu] = {} - pid_ram: dict[int, _PidRam] = {} - epochs: set[int] = set() - log_size = max(log.stat().st_size, 1) - bytes_seen = 0 - # Empirical: `atop -P PRC,PRM` stdout is ~11x the binary log size on a - # 10-min-interval log. The fraction is only used for the progress bar, - # so a rough calibration is fine; it caps at 99% if we underestimate. - est_total_bytes = log_size * 11 or 1 - for raw in _iter_atop_lines(log, "PRC,PRM", begin, end): - bytes_seen += len(raw) + 1 - if not raw or raw[0] == "#" or raw.startswith("RESET") or raw == "SEP": - continue - parts = raw.split() - if not parts: - continue - label = parts[0] - if label == "PRC" and len(parts) >= _PRC_MIN_LEN: - with contextlib.suppress(ValueError): - # atop always emits an integer epoch here; guard is defensive. - epochs.add(int(parts[2])) - progress.update(min(bytes_seen / est_total_bytes, 0.99)) - _parse_prc(parts, pid_cpu) - elif label == "PRM" and len(parts) >= _PRM_MIN_LEN: - _parse_prm(parts, pid_ram) - progress.update(1.0) - return _fold_pid_aggregates(pid_cpu, pid_ram), _window_from_epochs(epochs) - - -def _fold_pid_aggregates( - pid_cpu: dict[int, _PidCpu], - pid_ram: dict[int, _PidRam], -) -> dict[str, ProcAgg]: - """Collapse per-PID CPU/RAM trackers into per-program `ProcAgg` entries.""" - agg: dict[str, ProcAgg] = {} - for pid, cpu in pid_cpu.items(): - entry = agg.setdefault(cpu.name, ProcAgg(name=cpu.name)) - entry.cpu_ticks += cpu.delta_ticks - entry.pid_set.add(pid) - for pid, ram in pid_ram.items(): - entry = agg.setdefault(ram.name, ProcAgg(name=ram.name)) - entry.peak_rss_kb = max(entry.peak_rss_kb, ram.peak_kb) - entry.rss_kb_sum += int(ram.avg_kb) - entry.rss_samples += 1 - entry.pid_set.add(pid) - return agg - - -def merge_proc_aggs(dst: dict[str, ProcAgg], src: dict[str, ProcAgg]) -> None: - """Fold one day's CPU/RAM aggregates (*src*) into the running *dst*. - - CPU-seconds and RSS sample counts add across days; peak RSS is the max; - PID counts add (each day contributes its own distinct PIDs). - """ - for name, item in src.items(): - entry = dst.setdefault(name, ProcAgg(name=name)) - entry.cpu_ticks += item.cpu_ticks - entry.peak_rss_kb = max(entry.peak_rss_kb, item.peak_rss_kb) - entry.rss_kb_sum += item.rss_kb_sum - entry.rss_samples += item.rss_samples - entry.extra_pids += item.pid_count - - -def merge_gpu_aggs(dst: dict[str, GpuAgg], src: dict[str, GpuAgg]) -> None: - """Fold one day's GPU aggregates (*src*) into the running *dst*.""" - for name, item in src.items(): - entry = dst.setdefault(name, GpuAgg(name=name)) - entry.sm_pct_sum += item.sm_pct_sum - entry.mem_pct_sum += item.mem_pct_sum - entry.samples += item.samples - entry.peak_sm_pct = max(entry.peak_sm_pct, item.peak_sm_pct) - entry.peak_mem_pct = max(entry.peak_mem_pct, item.peak_mem_pct) - entry.extra_pids += item.pid_count - - -def merge_windows(windows: list[_Window]) -> _Window: - """Combine per-day coverage *windows* into one spanning window. - - Start/end span the earliest and latest samples; ``seconds`` sums the - per-day coverage (not wall-clock end-start) so the denominator for average - CPU% reflects only the time actually monitored, excluding gap days. - """ - real = [w for w in windows if w.distinct_samples] - if not real: - return _Window() - first = min(real, key=lambda w: w.start_epoch) - last = max(real, key=lambda w: w.end_epoch) - intervals = [w.interval_s for w in real if w.interval_s] - # Representative interval = the most common per-day interval, if any. - interval = max(set(intervals), key=intervals.count) if intervals else 0 - return _Window( - start=first.start, - end=last.end, - distinct_samples=sum(w.distinct_samples for w in real), - interval_s=interval, - seconds=sum(w.seconds for w in real), - start_epoch=first.start_epoch, - end_epoch=last.end_epoch, - ) diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_pmon.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_pmon.py deleted file mode 100644 index c697bbfb..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_pmon.py +++ /dev/null @@ -1,128 +0,0 @@ -"""nvidia-smi pmon log parsing and aggregation helpers for usage_report.""" - -from __future__ import annotations - -import datetime as _dt -from pathlib import Path - -from _usage_report_types import GpuAgg, _Progress - -_PMON_MIN_FIELDS = 11 - - -def _pmon_fields(line: str) -> list[str] | None: - """Return stripped fields of a pmon data line, or None for headers/blanks.""" - s = line.strip() - if not s or s.startswith("#"): - return None - return s.split() - - -def _normalize_pmon_command(command_fields: list[str]) -> str: - """Normalize pmon command fields into a stable process-ish name. - - `nvidia-smi pmon -o DT` emits fixed numeric columns followed by a command - field that can include whitespace. We prefer the *first* non-option token - (usually executable) and normalize it to a basename. - """ - tokens = [token.strip().strip("\"'") for token in command_fields if token.strip()] - if not tokens: - return "unknown" - - selected = tokens[0] - if selected.startswith("-"): - for candidate in tokens[1:]: - if not candidate.startswith("-"): - selected = candidate - break - - name = Path(selected).name.strip(";,:") - if not name: - return "unknown" - return name - - -def _pid_comm_name(pid: int) -> str | None: - """Return `/proc//comm` basename when available.""" - try: - comm = Path(f"/proc/{pid}/comm").read_text(encoding="utf-8").strip() - except OSError: - return None - return Path(comm).name if comm else None - - -def _pmon_row_epoch(parts: list[str]) -> float | None: - """Local-time epoch of a pmon row from its `date`/`time` columns, or None. - - pmon timestamps are naive local time (`YYYYMMDD HH:MM:SS`); `.astimezone()` - attaches the local offset so the result is comparable to a `begin_epoch` - derived the same way. - """ - try: - stamp = _dt.datetime.strptime( - f"{parts[0]} {parts[1]}", - "%Y%m%d %H:%M:%S", - ).astimezone() - except (ValueError, IndexError): - return None - return stamp.timestamp() - - -def aggregate_pmon( - log: Path, - progress: _Progress, - begin_epoch: float | None = None, -) -> tuple[dict[str, GpuAgg], int]: - """Return `({program: GpuAgg}, sample_count)` from the pmon *log*. - - When *begin_epoch* is set, rows timestamped before it are skipped so the - first day of a "since last report" window starts at the previous run time. - """ - progress.start_stage("pmon log scan") - agg: dict[str, GpuAgg] = {} - samples = 0 - if not log.exists(): - progress.update(1.0) - return agg, 0 - total_bytes = max(log.stat().st_size, 1) - bytes_read = 0 - with log.open(encoding="utf-8") as fh: - for line in fh: - bytes_read += len(line) - progress.update(min(bytes_read / total_bytes, 0.99)) - parts = _pmon_fields(line) - if parts is None or len(parts) < _PMON_MIN_FIELDS: - continue - if begin_epoch is not None: - row_epoch = _pmon_row_epoch(parts) - if row_epoch is not None and row_epoch < begin_epoch: - continue - samples += _ingest_pmon_row(parts, agg) - progress.update(1.0) - return agg, samples - - -def _ingest_pmon_row(parts: list[str], agg: dict[str, GpuAgg]) -> int: - """Fold a single pmon data row into *agg*; return 1 if consumed else 0.""" - # pmon -o DT fields: - # date time gpu pid type sm mem enc dec jpg ofa command - try: - pid = int(parts[3]) - except ValueError: - return 0 - sm_raw = parts[5] - mem_raw = parts[6] - command_fields = parts[11:] - name = _normalize_pmon_command(command_fields) - if name == "unknown": - name = _pid_comm_name(pid) or "unknown" - sm = float(sm_raw) if sm_raw != "-" else 0.0 - mem = float(mem_raw) if mem_raw != "-" else 0.0 - entry = agg.setdefault(name, GpuAgg(name=name)) - entry.sm_pct_sum += sm - entry.mem_pct_sum += mem - entry.samples += 1 - entry.pid_set.add(pid) - entry.peak_sm_pct = max(entry.peak_sm_pct, sm) - entry.peak_mem_pct = max(entry.peak_mem_pct, mem) - return 1 diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_render.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_render.py deleted file mode 100644 index 038d5e90..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_render.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Markdown report assembly for usage_report (host, methodology, full report).""" - -from __future__ import annotations - -import datetime as _dt -import os -from pathlib import Path -import platform -import re -from typing import TYPE_CHECKING - -from _usage_report_atop import _run -from _usage_report_format import _fmt_h -from _usage_report_tables import _cpu_table, _gpu_table, _ram_table -from _usage_report_types import _HZ, _PMON_INTERVAL_S, _Window - -if TYPE_CHECKING: - from usage_report import _Aggregates - -_PAGE_KB = os.sysconf("SC_PAGESIZE") // 1024 if hasattr(os, "sysconf") else 4 - - -def _host_profile() -> dict[str, str]: - """Collect a small bag of identifying facts about the host.""" - info: dict[str, str] = { - "hostname": platform.node(), - "kernel": platform.release(), - "cpus_online": str(os.cpu_count() or 0), - } - try: - with Path("/proc/cpuinfo").open(encoding="utf-8") as fh: - for line in fh: - if line.startswith("model name"): - info["cpu_model"] = line.split(":", 1)[1].strip() - break - except OSError: - pass - try: - with Path("/proc/meminfo").open(encoding="utf-8") as fh: - for line in fh: - if line.startswith("MemTotal:"): - kb = int(re.findall(r"\d+", line)[0]) - info["memory_total_gib"] = f"{kb / 1024 / 1024:.1f}" - break - except (OSError, IndexError, ValueError): - pass - gpu = _run( - [ - "nvidia-smi", - "--query-gpu=name,memory.total", - "--format=csv,noheader", - ], - ).strip() - if gpu: - info["gpu"] = gpu.replace("\n", "; ") - return info - - -def _fingerprint_section() -> list[str]: - info = _host_profile() - return [ - "## Host", - "", - *[f"- **{k}**: {v}" for k, v in info.items()], - "", - ] - - -def _methodology_section( - atop_desc: str, - pmon_desc: str, - window: _Window, -) -> list[str]: - window_note = ( - f"- **Coverage window**: {_fmt_h(window.seconds)} " - f"(sum of per-day atop coverage from first to last sample; excludes " - f"any gap days where atop was not logging, and the final partial tick)." - ) - interval_note = ( - f"- **atop sample interval (observed)**: {window.interval_s}s" - if window.interval_s - else "- **atop sample interval**: only one sample so far; interval unknown." - ) - task_note = ( - "- atop's parseable output is **task-level** (threads get their own " - "rows keyed by `/proc//comm`); names like 'Main Thread' or " - "'dxvk-frame' are usually Wine/game worker threads of one parent." - ) - rss_note = ( - "- RSS is shared across threads of one process, so multiple rows " - "with identical 'Peak RSS' almost certainly belong to a single " - "parent. The RAM table dedupes by peak-RSS bucket and lists " - "sibling thread names under `(+ siblings)`." - ) - cpu_note = ( - "- **CPU-seconds** are computed per-PID as " - "`last_cumulative_ticks - first_cumulative_ticks` (or the cumulative " - "value itself for PIDs seen only once). They reflect CPU consumed " - "during the coverage window only, not since process start." - ) - gpu_note = ( - "- GPU SM-seconds = sum(sm% per sample) \u00d7 sample interval / 100; " - "single-GPU equivalent." - ) - prog_note = ( - "- 'Program' = executable/thread name; rows with the same name " - "are summed across their distinct PIDs." - ) - return [ - "## Methodology", - "", - f"- **atop log(s)**: {atop_desc}", - f"- **pmon log(s)**: {pmon_desc}", - f"- **HZ**: {_HZ} ticks/s; **page size**: {_PAGE_KB} KiB", - window_note, - interval_note, - cpu_note, - task_note, - rss_note, - gpu_note, - prog_note, - "", - ] - - -_LLM_PROMPT = [ - "> Below is aggregated resource usage for my Linux workstation over the", - "> reporting period shown above. Identify which programs are the biggest", - "> hogs, flag anything that looks abnormal for a typical developer/gaming", - "> setup, and suggest concrete optimisations (config tweaks, process limits,", - "> alternative tools). Be specific.", -] - - -def _render_report( - aggs: _Aggregates, - *, - top: int, - atop_desc: str, - pmon_desc: str, - period_line: str, -) -> str: - """Assemble the Markdown report from already-aggregated data.""" - window = aggs.window - gpu_section = ( - _gpu_table(aggs.gpu, aggs.gpu_samples, top) - if aggs.gpu - else ["_No GPU pmon data found._"] - ) - generated = _dt.datetime.now().astimezone().isoformat(timespec="seconds") - interval = f"{window.interval_s}s" if window.interval_s else "n/a (single sample)" - lines: list[str] = [ - "# System resource usage report", - "", - f"- **Generated**: {generated}", - period_line, - f"- **atop window**: {window.start} → {window.end}", - f"- **atop samples**: {window.distinct_samples} distinct " - f"timestamps (sample interval ≈ {interval})", - f"- **GPU pmon samples**: {aggs.gpu_samples} (≈{_PMON_INTERVAL_S}s each)", - "", - *_fingerprint_section(), - *_methodology_section(atop_desc, pmon_desc, window), - "## Top CPU consumers", - "", - *_cpu_table(aggs.cpu.values(), window.seconds, top), - "", - "## Top RAM consumers (by peak RSS, deduped by shared-memory bucket)", - "", - *_ram_table(aggs.cpu.values(), top), - "", - "## Top GPU consumers", - "", - *gpu_section, - "", - "## Suggested LLM prompt", - "", - *_LLM_PROMPT, - "", - ] - return "\n".join(lines) + "\n" diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_run.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_run.py deleted file mode 100644 index a726522c..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_run.py +++ /dev/null @@ -1,182 +0,0 @@ -"""The two reporting runs usage_report dispatches between. - -`_run_single_day` covers one calendar day ad hoc; `_run_since` spans every -daily log from the last report to now and advances the saved timestamp. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import datetime as _dt -import shutil -import sys -from typing import TYPE_CHECKING - -from _usage_report_emit import _INSTALL_SCRIPT, _emit, _preflight -from _usage_report_format import _fmt_h -from _usage_report_logs import ( - _log_descriptions, - _plan_segments, - _read_last_generated, - _resolve_logs, - _Segment, - _write_last_generated, -) -from _usage_report_parsing import ( - aggregate_atop, - merge_gpu_aggs, - merge_proc_aggs, - merge_windows, -) -from _usage_report_pmon import aggregate_pmon -from _usage_report_render import _render_report -from _usage_report_types import GpuAgg, ProcAgg, _Progress, _Window - -_SEC_PER_DAY = 86_400 - -if TYPE_CHECKING: - import argparse - - -_REPORT_STAGES = 2 - - -@dataclass -class _Aggregates: - """Merged CPU/GPU aggregates and coverage window for a reporting window. - - *days_with_data* is the number of daily logs that actually yielded atop - samples (gap days where the machine was off contribute nothing). - """ - - cpu: dict[str, ProcAgg] - gpu: dict[str, GpuAgg] - window: _Window - gpu_samples: int - days_with_data: int - - -def _aggregate_segments( - segments: list[_Segment], - progress: _Progress, -) -> _Aggregates: - """Aggregate and merge every existing daily log in *segments*. - - Missing daily logs (gap days) are skipped silently. - """ - cpu_total: dict[str, ProcAgg] = {} - gpu_total: dict[str, GpuAgg] = {} - windows: list[_Window] = [] - gpu_samples = 0 - days_with_data = 0 - for seg in segments: - if seg.atop_log.exists(): - cpu, window = aggregate_atop(seg.atop_log, progress, seg.atop_begin) - merge_proc_aggs(cpu_total, cpu) - if window.distinct_samples: - windows.append(window) - days_with_data += 1 - gpu, samples = aggregate_pmon(seg.pmon_log, progress, seg.pmon_begin_epoch) - merge_gpu_aggs(gpu_total, gpu) - gpu_samples += samples - return _Aggregates( - cpu_total, - gpu_total, - merge_windows(windows), - gpu_samples, - days_with_data, - ) - - -def _period_line(start: _dt.datetime, end: _dt.datetime) -> str: - """Markdown bullet describing the requested reporting period.""" - span = _fmt_h(max((end - start).total_seconds(), 0.0)) - return ( - f"- **Reporting period**: {start.isoformat(timespec='seconds')} → " - f"{end.isoformat(timespec='seconds')} ({span})" - ) - - -def _should_advance_state(args: argparse.Namespace) -> bool: - """Advance the saved timestamp only for genuine since-last-report runs. - - An explicit ``--since`` is treated as a read-only ad-hoc query (like - ``--date``) so "let me look from date X" never silently re-baselines the - saved tracking point. - """ - return args.since is None and not args.no_update_state - - -def _run_single_day(args: argparse.Namespace, now: _dt.datetime) -> int: - """Report on one specific day (legacy behaviour); never touches state.""" - date = args.date or now.strftime("%Y%m%d") - atop_default, pmon_default = _resolve_logs(date) - atop_log = args.atop_log or atop_default - pmon_log = args.pmon_log or pmon_default - _preflight(atop_log) - segment = _Segment(atop_log, pmon_log) - progress = _Progress(enabled=not args.quiet, total_stages=_REPORT_STAGES) - aggs = _aggregate_segments([segment], progress) - progress.finish() - if not aggs.window.seconds: - aggs.window.seconds = _SEC_PER_DAY - atop_desc, pmon_desc = _log_descriptions([segment]) - _emit( - args, - _render_report( - aggs, - top=args.top, - atop_desc=atop_desc, - pmon_desc=pmon_desc, - period_line=f"- **Reporting period**: {date} (single day)", - ), - ) - return 0 - - -def _resolve_start(args: argparse.Namespace, now: _dt.datetime) -> _dt.datetime: - """Pick the window start: --since, else last report, else today midnight.""" - if args.since is not None: - return _dt.datetime.strptime(args.since, "%Y%m%d").astimezone() - last = _read_last_generated() - if last is not None: - return last - return now.replace(hour=0, minute=0, second=0, microsecond=0) - - -def _run_since(args: argparse.Namespace, now: _dt.datetime) -> int: - """Report on everything since the last run, spanning multiple daily logs.""" - if not shutil.which("atop"): - sys.exit(f"error: `atop` is not installed.\nrun: {_INSTALL_SCRIPT}") - start = _resolve_start(args, now) - segments = _plan_segments(start, now) - progress = _Progress( - enabled=not args.quiet, - total_stages=max(2 * len(segments), 1), - ) - aggs = _aggregate_segments(segments, progress) - progress.finish() - if aggs.days_with_data == 0: - sys.stderr.write( - f"no atop logs with data for {start.date()} … {now.date()}; " - "nothing to report.\n", - ) - if _should_advance_state(args): - _write_last_generated(now) - return 0 - if not aggs.window.seconds: - aggs.window.seconds = _SEC_PER_DAY - atop_desc, pmon_desc = _log_descriptions(segments) - _emit( - args, - _render_report( - aggs, - top=args.top, - atop_desc=atop_desc, - pmon_desc=pmon_desc, - period_line=_period_line(start, now), - ), - ) - if _should_advance_state(args): - _write_last_generated(now) - return 0 diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_tables.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_tables.py deleted file mode 100644 index bb6c0e44..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_tables.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Markdown table builders for usage_report (CPU, RAM and GPU sections).""" - -from __future__ import annotations - -from collections import defaultdict -import os -from typing import TYPE_CHECKING - -from _usage_report_format import _fmt_h, _md_escape - -if TYPE_CHECKING: - from collections.abc import Iterable - - from _usage_report_types import GpuAgg, ProcAgg - -_RAM_BUCKET_MIB = 1 # dedupe rows whose peak RSS rounds to the same MiB -_MAX_SIBLINGS_SHOWN = 6 - - -def _cpu_table(aggs: Iterable[ProcAgg], window_s: int, top: int) -> list[str]: - ncpu = os.cpu_count() or 1 - header = ( - "| # | Program | CPU-seconds | Avg CPU% (of 1 core) |" - " Avg CPU% (of box) | Peak RSS | PIDs |" - ) - sep = ( - "|---|---------|------------:|---------------------:|" - "------------------:|---------:|-----:|" - ) - rows: list[str] = [header, sep] - top_items = sorted(aggs, key=lambda a: a.cpu_ticks, reverse=True)[:top] - for idx, item in enumerate(top_items, start=1): - single = (item.cpu_seconds / window_s * 100) if window_s else 0.0 - box = single / ncpu - rows.append( - "| " - f"{idx} | {_md_escape(item.name)} | " - f"{item.cpu_seconds:,.0f}s ({_fmt_h(item.cpu_seconds)}) | " - f"{single:.1f}% | {box:.1f}% | " - f"{item.peak_rss_mb:,.0f} MiB | {item.pid_count} |", - ) - return rows - - -def _dedupe_ram(aggs: Iterable[ProcAgg]) -> list[tuple[ProcAgg, list[str]]]: - """Group rows by peak-RSS bucket; keep the top-CPU row per bucket. - - Returns a list of `(representative, sibling_names)` ordered by peak RSS - descending. Siblings are the other names that shared the same RSS bucket - (likely threads of the same parent process). - """ - buckets: dict[int, list[ProcAgg]] = defaultdict(list) - for item in aggs: - if item.peak_rss_kb <= 0: - continue - key = round(item.peak_rss_kb / 1024 / _RAM_BUCKET_MIB) - buckets[key].append(item) - result: list[tuple[ProcAgg, list[str]]] = [] - for bucket in buckets.values(): - bucket.sort(key=lambda a: (a.cpu_ticks, a.pid_count), reverse=True) - rep = bucket[0] - siblings = [b.name for b in bucket[1:]] - result.append((rep, siblings)) - result.sort(key=lambda t: t[0].peak_rss_kb, reverse=True) - return result - - -def _ram_table(aggs: Iterable[ProcAgg], top: int) -> list[str]: - header = ( - "| # | Program | Peak RSS | Avg RSS | CPU-seconds | PIDs |" - " Sibling names (shared RSS) |" - ) - sep = ( - "|---|---------|---------:|--------:|------------:|-----:|" - "----------------------------|" - ) - rows: list[str] = [header, sep] - for idx, (item, siblings) in enumerate(_dedupe_ram(aggs)[:top], start=1): - if not siblings: - sib = "—" - else: - shown = ", ".join(_md_escape(s) for s in siblings[:_MAX_SIBLINGS_SHOWN]) - extra = ( - f" (+{len(siblings) - _MAX_SIBLINGS_SHOWN} more)" - if len(siblings) > _MAX_SIBLINGS_SHOWN - else "" - ) - sib = f"{shown}{extra}" - rows.append( - "| " - f"{idx} | {_md_escape(item.name)} | " - f"{item.peak_rss_mb:,.0f} MiB | " - f"{item.avg_rss_mb:,.0f} MiB | " - f"{item.cpu_seconds:,.0f}s | " - f"{item.pid_count} | {sib} |", - ) - return rows - - -def _gpu_table(aggs: dict[str, GpuAgg], total_samples: int, top: int) -> list[str]: - header = ( - "| # | Program | GPU SM-seconds | Avg SM% (when present) |" - " Peak SM% | Peak MEM% | Samples | PIDs |" - ) - sep = ( - "|---|---------|---------------:|-----------------------:|" - "---------:|----------:|--------:|-----:|" - ) - rows: list[str] = [header, sep] - top_items = sorted(aggs.values(), key=lambda a: a.gpu_seconds, reverse=True)[:top] - for idx, item in enumerate(top_items, start=1): - presence = (item.samples / total_samples * 100) if total_samples else 0.0 - rows.append( - "| " - f"{idx} | {_md_escape(item.name)} | " - f"{item.gpu_seconds:,.0f}s ({_fmt_h(item.gpu_seconds)}) | " - f"{item.avg_sm_pct:.1f}% | " - f"{item.peak_sm_pct:.0f}% | " - f"{item.peak_mem_pct:.0f}% | " - f"{item.samples} ({presence:.0f}%) | " - f"{item.pid_count} |", - ) - return rows diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_types.py b/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_types.py deleted file mode 100644 index 9386e6a0..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/_usage_report_types.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Shared data-class types and progress reporter for usage_report.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -import os -import sys -import time as _time - -_HZ = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100 -_MIN_SAMPLES_FOR_WINDOW = 2 -# Default pmon interval is 10 s (matches the systemd service we set up). -_PMON_INTERVAL_S = 10 -_PROGRESS_MIN_UPDATE_S = 0.1 -_ETA_MIN_FRACTION = 0.01 - - -@dataclass -class _PidCpu: - """Per-PID cumulative-ticks tracker across atop samples.""" - - name: str = "" - first_ticks: int = -1 - last_ticks: int = 0 - samples: int = 0 - - def observe(self, name: str, ticks: int) -> None: - """Record one observation for this PID.""" - self.name = name # last-seen name wins (stable for one PID) - if self.first_ticks < 0: - self.first_ticks = ticks - self.last_ticks = ticks - self.samples += 1 - - @property - def delta_ticks(self) -> int: - """CPU ticks consumed during the observation window. - - For PIDs seen in >=2 samples the value is `last - first`, which is the - actual CPU consumed between the first and last atop tick. For PIDs seen - only once (short-lived processes that existed during exactly one tick) - the cumulative value itself is used — this is close to the true - lifetime cost for a short-lived process. - """ - if self.samples >= _MIN_SAMPLES_FOR_WINDOW: - return max(self.last_ticks - self.first_ticks, 0) - return self.last_ticks - - -@dataclass -class _PidRam: - """Per-PID peak/avg RSS tracker across atop samples.""" - - name: str = "" - peak_kb: int = 0 - sum_kb: int = 0 - samples: int = 0 - - def observe(self, name: str, rss_kb: int) -> None: - """Record one RSS observation for this PID.""" - self.name = name - self.peak_kb = max(self.peak_kb, rss_kb) - self.sum_kb += rss_kb - self.samples += 1 - - @property - def avg_kb(self) -> float: - """Mean RSS across the samples where this PID appeared.""" - return self.sum_kb / self.samples if self.samples else 0.0 - - -@dataclass -class ProcAgg: - """Aggregated metrics for one program name across all atop samples.""" - - name: str - cpu_ticks: int = 0 - peak_rss_kb: int = 0 - rss_kb_sum: int = 0 - rss_samples: int = 0 - pid_set: set[int] = field(default_factory=set) - # PID counts folded in when merging per-day aggregates. Tracked as a plain - # integer (not by extending `pid_set`) because the native parser stores a - # synthetic `range(n)` set whose union across days would collapse counts. - extra_pids: int = 0 - - @property - def pid_count(self) -> int: - """Distinct PIDs seen, including those merged from other day windows.""" - return len(self.pid_set) + self.extra_pids - - @property - def cpu_seconds(self) -> float: - """CPU-seconds consumed (sum of user + system time).""" - return self.cpu_ticks / _HZ - - @property - def peak_rss_mb(self) -> float: - """Peak resident memory observed across the window, in MiB.""" - return self.peak_rss_kb / 1024 - - @property - def avg_rss_mb(self) -> float: - """Average resident memory across samples where the program appeared.""" - if not self.rss_samples: - return 0.0 - return (self.rss_kb_sum / self.rss_samples) / 1024 - - -@dataclass -class GpuAgg: - """Aggregated GPU metrics for one program name from pmon logs.""" - - name: str - sm_pct_sum: float = 0.0 - mem_pct_sum: float = 0.0 - samples: int = 0 - peak_sm_pct: float = 0.0 - peak_mem_pct: float = 0.0 - pid_set: set[int] = field(default_factory=set) - # PID counts folded in when merging per-day aggregates (see ProcAgg). - extra_pids: int = 0 - - @property - def pid_count(self) -> int: - """Distinct PIDs seen, including those merged from other day windows.""" - return len(self.pid_set) + self.extra_pids - - @property - def gpu_seconds(self) -> float: - """SM-seconds (single-GPU equivalent); sm% * seconds_per_sample / 100.""" - return self.sm_pct_sum * _PMON_INTERVAL_S / 100.0 - - @property - def avg_sm_pct(self) -> float: - """Mean SM utilization across samples where the process was present.""" - if not self.samples: - return 0.0 - return self.sm_pct_sum / self.samples - - -class _Progress: - """Minimal stage+percent+ETA reporter on stderr. - - Disabled automatically when stderr is not a TTY or when the caller - constructs with `enabled=False`, so redirected output stays clean. - """ - - def __init__(self, *, enabled: bool, total_stages: int) -> None: - self._enabled = enabled and sys.stderr.isatty() - self._total_stages = total_stages - self._stage_idx = 0 - self._stage_label = "" - self._stage_start = 0.0 - self._t0 = _time.monotonic() - self._last_draw = 0.0 - self._max_width = 0 - - def start_stage(self, label: str) -> None: - """Begin a new stage with its human label.""" - self._stage_idx += 1 - self._stage_label = label - self._stage_start = _time.monotonic() - self.update(0.0) - - def update(self, fraction: float) -> None: - """Redraw the progress line for the current stage (0.0..1.0).""" - if not self._enabled: - return - now = _time.monotonic() - if now - self._last_draw < _PROGRESS_MIN_UPDATE_S and fraction < 1.0: - return - self._last_draw = now - elapsed = now - self._stage_start - pct = max(0.0, min(fraction, 1.0)) - if pct > _ETA_MIN_FRACTION: - eta = elapsed * (1 - pct) / pct - eta_str = f"~{eta:4.1f}s left" - else: - eta_str = "estimating…" - msg = ( - f"[{self._stage_idx}/{self._total_stages}] " - f"{self._stage_label:<22} {pct * 100:5.1f}% " - f"{elapsed:5.1f}s elapsed, {eta_str}" - ) - self._max_width = max(self._max_width, len(msg)) - sys.stderr.write("\r" + msg.ljust(self._max_width)) - sys.stderr.flush() - - def finish(self) -> None: - """Clear the progress line and print total elapsed time.""" - if not self._enabled: - return - total = _time.monotonic() - self._t0 - sys.stderr.write("\r" + " " * self._max_width + "\r") - sys.stderr.write(f"done in {total:.1f}s\n") - sys.stderr.flush() - - -@dataclass -class _Window: - """Observed atop coverage window.""" - - start: str = "n/a" - end: str = "n/a" - distinct_samples: int = 0 - interval_s: int = 0 - seconds: int = 0 - # Raw epoch bounds, kept so multiple per-day windows can be merged by - # min(start)/max(end) without re-parsing the ISO strings above. - start_epoch: int = 0 - end_epoch: int = 0 diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/auto-system-update.sh b/linux_configuration/scripts/periodic_background/system-maintenance/bin/auto-system-update.sh deleted file mode 100755 index f38632e3..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/auto-system-update.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# Automatic system update script for Arch Linux -# Runs pacman -Syuu and yay -Sua non-interactively -# This file is installed by setup_periodic_system.sh - -set -euo pipefail - -readonly LOG_FILE="/var/log/auto-system-update.log" -readonly LOCK_FILE="/var/lock/auto-system-update.lock" -readonly ACTUAL_USER="__ACTUAL_USER__" - -log_msg() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE" >&2 -} - -cleanup() { - rm -f "$LOCK_FILE" -} - -trap cleanup EXIT - -# Prevent concurrent runs -if ! (set -o noclobber && echo $$ > "$LOCK_FILE") 2>/dev/null; then - log_msg "Another update is already running (lock: $LOCK_FILE). Exiting." - exit 0 -fi - -log_msg "=== Automatic System Update Started ===" - -# --- Official repository update (pacman) --- -log_msg "Running pacman -Syuu --noconfirm ..." -if /usr/bin/pacman -Syuu --noconfirm >> "$LOG_FILE" 2>&1; then - log_msg "pacman update completed successfully" -else - log_msg "pacman update failed (exit $?)" -fi - -# --- AUR update (yay) --- -# yay must not run as root; run as the actual user -if command -v /usr/bin/yay > /dev/null 2>&1; then - log_msg "Running yay -Sua --noconfirm as $ACTUAL_USER ..." - if sudo -u "$ACTUAL_USER" /usr/bin/yay -Sua --noconfirm 2>&1 | tee -a "$LOG_FILE" > /dev/null; then - log_msg "yay AUR update completed successfully" - else - log_msg "yay AUR update failed (exit $?)" - fi -else - log_msg "yay not found, skipping AUR updates" -fi - -log_msg "=== Automatic System Update Completed ===" diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh b/linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh deleted file mode 100755 index f3c79ad9..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash -# Generic pre-exec wrapper for browsers: ensures /etc/hosts is (re)installed -# before launching the actual browser binary, and — for Chromium-family -# browsers — captures a crash log for every launch. -# -# Why the crash logging exists (diagnosed 2026-08-09): -# Thorium was "closing and dying randomly" with NO evidence anywhere: -# Crashpad was never enabled (no crash_reporting consent key, empty crash DB), -# and stderr from a dmenu/.desktop launch goes to no terminal and no journal. -# The only Thorium lines ever seen in the journal came from a systemd-launched -# instance. So the browser could die repeatedly and leave nothing behind. -# -# Each launch now gets its own log holding: a VRAM/RAM snapshot before start, -# the browser's own stderr (--enable-logging=stderr --v=1), and — on exit — -# the exit code, decoded signal, and a SECOND VRAM snapshot. That last pair is -# the discriminator between the two live hypotheses: a GPU/VRAM death shows -# NVRM/NV_ERR_NO_MEMORY or near-full VRAM at exit, whereas a stale-binary -# death shows SIGSEGV/SIGILL with the GPU idle. - -set -euo pipefail - -HOSTS_INSTALL_SCRIPT="__HOSTS_INSTALL_SCRIPT__" - -# Where per-launch logs go, and how many to keep. -readonly LOG_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/browser-crash-logs" -readonly LOG_KEEP=20 - -# Per-log byte cap. Measured 2026-08-09: Thorium writes ~2 MB in the first 45s -# (65% of it VERBOSE1 net/TLS chatter) and ~0 while idle, so cost scales with -# navigation, not wall-clock. Capping COUNT alone is therefore not enough — a -# heavy browsing day could fill the disk, and a full disk causes exactly the -# kind of random death this logging exists to diagnose. -# -# Enforced by the watchdog below, which keeps the NEWEST bytes: the lines -# immediately before death are the diagnostic ones, so trimming must drop the -# head, never the tail. -readonly LOG_MAX_BYTES=$((32 * 1024 * 1024)) - -prog_name="$(basename "$0")" -real_bin="/usr/bin/${prog_name}" - -# If run directly (not via a browser symlink) or if the target binary doesn't exist, -# allow passing the real browser command as the first argument for testing: -if [[ ! -x $real_bin || $prog_name == "browser-preexec-wrapper.sh" ]]; then - if [[ $# -ge 1 ]]; then - real_bin="$1" - shift - else - echo "Error: could not resolve real browser binary for '$prog_name'." >&2 - echo "Usage (testing): $0 [args...]" >&2 - echo "Typical install: symlink this script as /usr/local/bin/ so it wraps /usr/bin/." >&2 - exit 127 - fi -fi - -# Best-effort: install hosts file quietly; don't block browser startup -if command -v sudo >/dev/null 2>&1; then - sudo -n "$HOSTS_INSTALL_SCRIPT" >/dev/null 2>&1 || true -else - "$HOSTS_INSTALL_SCRIPT" >/dev/null 2>&1 || true -fi - -# Only Chromium-family browsers understand --enable-logging/--v; everything else -# (firefox, librewolf) is exec'd unchanged so this wrapper stays generic. -is_chromium_family() { - case "$prog_name" in - thorium-browser | google-chrome | google-chrome-stable | chromium | brave | brave-browser | vivaldi-stable) - return 0 - ;; - *) return 1 ;; - esac -} - -# GPU + RAM state. Written before launch and again at exit so the two can be -# compared; nvidia-smi may be absent (or the GPU wedged), so never fail here. -emit_resource_snapshot() { - local label="$1" - echo "--- ${label} $(date '+%Y-%m-%d %H:%M:%S') ---" - if command -v nvidia-smi >/dev/null 2>&1; then - echo -n "VRAM used/total: " - nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader 2>&1 || echo "nvidia-smi failed" - else - echo "VRAM: nvidia-smi unavailable" - fi - echo -n "RAM: " - free -m | awk '/^Mem:/ {printf "%s MiB used / %s MiB total\n", $3, $2}' - echo "kernel: $(uname -r)" - # Match the dotted version anywhere on the line rather than by field index: - # the surrounding wording differs between open/proprietary driver builds. - # Worth logging because a module/userspace version skew (nvidia-utils upgraded - # without a reboot) is a known cause of GL crashes in Chromium. - if [[ -r /proc/driver/nvidia/version ]]; then - echo -n "nvidia module: " - grep -oE '[0-9]+\.[0-9]+\.[0-9]+' /proc/driver/nvidia/version 2>/dev/null | head -1 || echo "unreadable" - fi -} - -# Keep only the newest $keep logs. Uses -printf/sort -z so filenames containing -# spaces or newlines can't split a record. Called with LOG_KEEP-1 before a new -# log is created, so the directory settles at exactly LOG_KEEP afterwards. -rotate_logs() { - local keep="$1" - local old - while IFS= read -r -d '' old; do - rm -f -- "$old" - done < <( - find "$LOG_DIR" -maxdepth 1 -type f -name '*.log' -printf '%T@\t%p\0' 2>/dev/null | - sort -zrn | - tail -zn "+$((keep + 1))" | - cut -zf2- - ) -} - -if ! is_chromium_family; then - exec "$real_bin" "$@" -fi - -mkdir -p "$LOG_DIR" -rotate_logs "$((LOG_KEEP - 1))" -log_file="${LOG_DIR}/${prog_name}-$(date +%Y%m%d-%H%M%S)-$$.log" - -{ - echo "=== launch $(date '+%Y-%m-%d %H:%M:%S') ===" - echo "binary: $real_bin" - echo "args: $*" - emit_resource_snapshot "PRE-LAUNCH" - echo "=== browser stderr follows ===" -} >"$log_file" - -# Written from a trap rather than inline: if the wrapper itself is killed (the -# session ends, the process group is torn down, the OOM killer picks it), an -# inline block after `wait` would never run and the log would end mid-stream -# with no verdict — indistinguishable from a browser still running. -record_exit() { - local code="$1" - # Fire once; EXIT still runs after a signal trap has already reported. - [[ -n ${exit_recorded:-} ]] && return 0 - exit_recorded=1 - { - echo "=== exit ===" - echo "exit code: $code" - if ((code > 128)); then - local signal_num=$((code - 128)) - echo "died on signal: ${signal_num} ($(kill -l "$signal_num" 2>/dev/null || echo unknown))" - fi - emit_resource_snapshot "POST-EXIT" - } >>"$log_file" -} - -# Run in the background rather than exec'ing, so this shell survives to record -# how the browser died. `wait` reports 128+N for a signal death, which is the -# single most useful fact the old setup threw away. -# -# Deliberately a direct redirect, NOT a pipe into a size-limiting filter: in a -# pipeline `wait` reports the LAST element's status, which would report the -# filter's exit and discard the browser's signal — the one fact this whole -# wrapper exists to capture. Size is bounded by the watchdog below instead. -"$real_bin" --enable-logging=stderr --v=1 "$@" >>"$log_file" 2>&1 & -browser_pid=$! - -# Watchdog: if a browsing session makes the log exceed the cap, keep the most -# recent half and continue appending. Checks once a minute — the measured burst -# rate (~2 MB/45s of navigation, ~0 idle) can't overshoot 32 MB meaningfully in -# that window, and a 1/min poll costs nothing next to the browser itself. -( - while kill -0 "$browser_pid" 2>/dev/null; do - sleep 60 - size="$(stat -c %s "$log_file" 2>/dev/null || echo 0)" - if ((size > LOG_MAX_BYTES)); then - tail -c "$((LOG_MAX_BYTES / 2))" "$log_file" >"${log_file}.trim" 2>/dev/null && - mv "${log_file}.trim" "$log_file" && - echo "=== log trimmed to newest $((LOG_MAX_BYTES / 2)) bytes at $(date '+%H:%M:%S') ===" >>"$log_file" - fi - done -) & -watchdog_pid=$! - -# A signal aimed at the wrapper is recorded as the wrapper's own death, and the -# browser is taken down with it so no orphan outlives the log describing it. -# Written inline (not via a helper) so shellcheck can see the calls; each then -# re-raises with the default handler so the exit status stays truthful. -trap 'record_exit 129; kill "$browser_pid" "$watchdog_pid" 2>/dev/null || true; trap - HUP; kill -s HUP $$' HUP -trap 'record_exit 130; kill "$browser_pid" "$watchdog_pid" 2>/dev/null || true; trap - INT; kill -s INT $$' INT -trap 'record_exit 143; kill "$browser_pid" "$watchdog_pid" 2>/dev/null || true; trap - TERM; kill -s TERM $$' TERM - -exit_code=0 -wait "$browser_pid" || exit_code=$? - -kill "$watchdog_pid" 2>/dev/null || true -record_exit "$exit_code" -exit "$exit_code" diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/hosts-file-monitor.sh b/linux_configuration/scripts/periodic_background/system-maintenance/bin/hosts-file-monitor.sh deleted file mode 100755 index 5690bf9b..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/hosts-file-monitor.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/bash -# Hosts file monitor script -# Watches /etc/hosts for changes and restores it if needed -# This file is installed by setup_periodic_system.sh - -set -euo pipefail - -LOG_FILE="/var/log/hosts-file-monitor.log" -HOSTS_FILE="/etc/hosts" -HOSTS_INSTALL_SCRIPT="__HOSTS_INSTALL_SCRIPT__" -readonly MIN_HOSTS_LINES=1000 -readonly EVENT_COOLDOWN_S=5 - -current_epoch() { - local out_var="${1:-}" - if [[ -n $out_var ]]; then - printf -v "$out_var" '%(%s)T' -1 - else - printf '%(%s)T\n' -1 - fi -} - -wait_seconds() { - local timeout_s=$1 - local start_ts end_ts elapsed_s remaining_s - - printf -v start_ts '%(%s)T' -1 - IFS= read -r -t "$timeout_s" || true - printf -v end_ts '%(%s)T' -1 - - elapsed_s=$((end_ts - start_ts)) - if (( elapsed_s < timeout_s )); then - remaining_s=$((timeout_s - elapsed_s)) - sleep "$remaining_s" - fi -} - -# Log with timestamp (hosts-file-monitor specific) -log_message() { - local _ts - local msg - printf -v _ts '%(%Y-%m-%d %H:%M:%S)T' -1 - printf -v msg '%s [hosts-monitor] %s' "$_ts" "$1" - printf '%s\n' "$msg" >&2 - printf '%s\n' "$msg" >> "$LOG_FILE" 2>/dev/null || true -} - -# Function to check if hosts file needs restoration -needs_restoration() { - # Check if file exists - if [[ ! -f $HOSTS_FILE ]]; then - return 0 # File missing, needs restoration - fi - - # Check if file is empty or too small (less than MIN_HOSTS_LINES indicates tampering) - local line_count=0 - local has_custom_entries=0 - local has_stevenblack_entries=0 - local line="" - - while IFS= read -r line || [[ -n $line ]]; do - line_count=$((line_count + 1)) - - if [[ $has_custom_entries -eq 0 && $line == *"Custom blocking entries"* ]]; then - has_custom_entries=1 - fi - - if [[ $has_stevenblack_entries -eq 0 && $line == *"StevenBlack"* ]]; then - has_stevenblack_entries=1 - fi - - if (( line_count >= MIN_HOSTS_LINES && has_custom_entries == 1 && has_stevenblack_entries == 1 )); then - return 1 # File seems intact - fi - done < "$HOSTS_FILE" - - if [[ $line_count -lt $MIN_HOSTS_LINES ]]; then - return 0 # File too small, likely tampered with - fi - - # Check if our custom entries are missing - if [[ $has_custom_entries -eq 0 ]]; then - return 0 # Our custom entries missing, needs restoration - fi - - # Check if StevenBlack entries are missing - if [[ $has_stevenblack_entries -eq 0 ]]; then - return 0 # StevenBlack entries missing, needs restoration - fi - - return 1 # File seems intact -} - -# Function to restore hosts file -restore_hosts_file() { - log_message "Hosts file modification detected - initiating restoration" - - if [[ -f $HOSTS_INSTALL_SCRIPT ]]; then - log_message "Running hosts installation script: $HOSTS_INSTALL_SCRIPT" - - if bash "$HOSTS_INSTALL_SCRIPT" >> "$LOG_FILE" 2>&1; then - log_message "Hosts file restoration completed successfully" - else - log_message "Hosts file restoration failed with exit code $?" - fi - else - log_message "ERROR: Hosts install script not found at $HOSTS_INSTALL_SCRIPT" - fi -} - -# Function to monitor with inotifywait -monitor_with_inotify() { - log_message "Starting hosts file monitoring with inotify" - local last_check_ts=0 - - # Monitor the hosts file and its directory for various events - inotifywait -m -e delete,move,modify,attrib,create --format '%w%f %e %T' --timefmt '%Y-%m-%d %H:%M:%S' "$HOSTS_FILE" /etc/ 2> /dev/null | - while read -r file event time; do - # Check if the event is related to our hosts file - if [[ $file == "$HOSTS_FILE" ]] || [[ $file == "/etc/hosts" ]]; then - local now_ts - current_epoch now_ts - if (( now_ts - last_check_ts < EVENT_COOLDOWN_S )); then - continue - fi - last_check_ts=$now_ts - - log_message "Event detected: $event on $file at $time" - - # Check if restoration is needed - if needs_restoration; then - restore_hosts_file - else - log_message "Hosts file check passed - no restoration needed" - fi - fi - done -} - -# Function to monitor with polling (fallback) -monitor_with_polling() { - log_message "Starting hosts file monitoring with polling (fallback method)" - - while true; do - if needs_restoration; then - restore_hosts_file - fi - - # Check every 30 seconds - wait_seconds 30 - done -} - -start_monitoring() { - log_message "=== Hosts File Monitor Started ===" - - if command -v inotifywait > /dev/null 2>&1; then - log_message "Using inotify for file monitoring" - monitor_with_inotify - else - log_message "inotify-tools not available, using polling method" - log_message "Consider installing inotify-tools for better performance: pacman -S inotify-tools" - monitor_with_polling - fi -} - -# Main execution -if [[ ${HOSTS_FILE_MONITOR_SKIP_MAIN:-0} -ne 1 ]]; then - start_monitoring -fi diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/install_usage_monitoring.sh b/linux_configuration/scripts/periodic_background/system-maintenance/bin/install_usage_monitoring.sh deleted file mode 100755 index fdfde53b..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/install_usage_monitoring.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Install and enable the resource-monitoring stack used by usage_report.py: -# atop -- daily CPU/RAM/disk history (systemd service + rotation) -# nvtop -- live GPU top (optional, NVIDIA/AMD/Intel) -# netdata -- live dashboard on http://localhost:19999 (optional) -# a clipboard tool (wl-clipboard or xclip) so usage_report.py can paste -# -# Plus an `nvidia-pmon` user service that logs per-process GPU samples to -# ~/.local/share/gpu-log/pmon-YYYYMMDD.log (only if nvidia-smi is present). -# -# Works on Arch, Debian/Ubuntu (and derivatives), Fedora/RHEL, openSUSE. -# Re-run safely; everything is idempotent. -# -# The steps live in lib/; this file owns the shared helpers, the globals that -# cross between steps (FAMILY, pkgs, clip, unit_dir, REPO_DIR) and the order. - -set -euo pipefail - -log() { printf '[install-usage] %s\n' "$*" >&2; } -die() { - printf '[install-usage] ERROR: %s\n' "$*" >&2 - exit 1 -} - -# readlink -f so a symlinked entry point still finds lib/. -SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -# shellcheck source=lib/distro_detect.sh -. "$SCRIPT_DIR/lib/distro_detect.sh" -# shellcheck source=lib/packages.sh -. "$SCRIPT_DIR/lib/packages.sh" -# shellcheck source=lib/system_services.sh -. "$SCRIPT_DIR/lib/system_services.sh" -# shellcheck source=lib/nvidia_pmon.sh -. "$SCRIPT_DIR/lib/nvidia_pmon.sh" -# shellcheck source=lib/catchup_timer.sh -. "$SCRIPT_DIR/lib/catchup_timer.sh" - -[[ $EUID -eq 0 ]] && die "run as your normal user; sudo is invoked where needed" -command -v sudo >/dev/null 2>&1 || die "sudo is required" - -detect_family -resolve_and_install_packages -enable_system_services -setup_nvidia_pmon - -# Resolved here, once, from the entry script's own location: lib/ sits one -# level deeper, so a lib recomputing this would point at the wrong repo. -REPO_DIR="$SCRIPT_DIR/../../../../.." -REPO_DIR="$(readlink -f "$REPO_DIR")" -setup_catchup_timer - -log "done. Wait for the first atop sample (default 10 min), then run:" -log " python $SCRIPT_DIR/usage_report.py" diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/lib/catchup_timer.sh b/linux_configuration/scripts/periodic_background/system-maintenance/bin/lib/catchup_timer.sh deleted file mode 100755 index 81d5d7e6..00000000 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/lib/catchup_timer.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Installs the hourly usage-report catch-up user timer. -# Sourced by install_usage_monitoring.sh; inherits the caller's strict mode. -# -# REPO_DIR is computed ONCE by the entry script and read here as a global. It -# is interpolated into the generated catch-up script at write time, so -# recomputing it from this file's own depth would silently point the generated -# script at the wrong repo while still exiting 0. - -setup_catchup_timer() { - unit_dir="$HOME/.config/systemd/user" - mkdir -p "$unit_dir" "$HOME/.local/bin" "$HOME/.local/share/usage-reports" - - # Unquoted delimiter: $HOME and $REPO_DIR interpolate at write time, while - # \$REPO and \$OUT_DIR stay literal. The escaping is load-bearing. - cat >"$HOME/.local/bin/usage-report-catchup.sh" <