diff --git a/docs/superpowers/evidence/thorium-crash-logging-2026-08-09.json b/docs/superpowers/evidence/thorium-crash-logging-2026-08-09.json new file mode 100644 index 00000000..e1545ac8 --- /dev/null +++ b/docs/superpowers/evidence/thorium-crash-logging-2026-08-09.json @@ -0,0 +1,65 @@ +{ + "intent": "Make Thorium (and every Chromium-family browser) record why it died, so 'the browser closes randomly' stops being undiagnosable. Previously there was no evidence at all: Crashpad was never enabled (no crash_reporting consent key, empty crash DB) and stderr from a dmenu/.desktop launch reached no terminal and no journal.", + "scope": [ + "linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh", + "linux_configuration/scripts/periodic_background/digital_wellbeing/pacman/pacman_whitelist.txt", + "Non-goal: fixing the crash itself — root cause is still unconfirmed by design; this change is what produces the evidence to confirm it.", + "Non-goal: changing behaviour for non-Chromium browsers (firefox/librewolf are exec'd unchanged)." + ], + "changes": [ + "browser-preexec-wrapper.sh: for Chromium-family names only, launch the browser in the background instead of exec'ing, so the wrapper survives to record how it died. Adds --enable-logging=stderr --v=1 and writes a per-launch log to $XDG_STATE_HOME/browser-crash-logs.", + "Each log records a PRE-LAUNCH and POST-EXIT snapshot (VRAM used/total, RAM, kernel, nvidia module version) plus the exit code and decoded signal. The two VRAM snapshots are the discriminator between the VRAM-exhaustion and stale-binary hypotheses.", + "Exit recording moved into record_exit() called from HUP/INT/TERM traps, so the verdict is written even when the wrapper itself is killed; an inline block after `wait` would silently leave the log ending mid-stream.", + "Count rotation (LOG_KEEP=20) plus a byte watchdog (LOG_MAX_BYTES=32 MiB) that trims to the newest half. Measured 2026-08-09: ~2 MB per 45s of navigation, ~0 while idle — count alone would not bound disk use, and a full disk causes exactly the random deaths under investigation.", + "Deliberately a direct redirect rather than a pipe into a size filter: in a pipeline `wait` reports the last element's status, which would discard the browser's signal — the one fact the wrapper exists to capture.", + "pacman_whitelist.txt: allow the exact package name `librewolf` (the `librewolf` keyword stays blocked, as do all other Firefox forks). LibreWolf is LeechBlock-enforced via /etc/librewolf/policies, so this is not an unfiltered escape hatch; needed because Chromium's MV2 sunset ends full uBlock Origin with all filter lists." + ], + "verification": [ + { + "command": "shellcheck + shfmt -w on browser-preexec-wrapper.sh", + "result": "pass", + "evidence": "shellcheck clean (no SC codes); shfmt applied repo tab style. No lint suppressions added." + }, + { + "command": "TEST A: fake browser killed with SIGSEGV under the wrapper", + "result": "pass", + "evidence": "log tail: 'exit code: 139' / 'died on signal: 11 (SEGV)' plus both VRAM snapshots." + }, + { + "command": "TEST B: wrapper itself SIGTERMed", + "result": "pass", + "evidence": "trap fired: 'exit code: 143' / 'died on signal: 15 (TERM)' — the case an inline post-wait block would have missed." + }, + { + "command": "TEST C: orphan check after both death modes", + "result": "pass", + "evidence": "no surviving fake-browser processes (initial FAIL traced to a leftover from an aborted setsid run started before the test, and to the pgrep pattern matching the test's own shell)." + }, + { + "command": "TEST D: trim watchdog with LOG_MAX_BYTES=4096 against a 4000-line spammer", + "result": "pass", + "evidence": "log bounded to 2101 bytes with 1 'log trimmed' marker; newest bytes retained." + }, + { + "command": "Real binary: kill -SEGV on live /opt/thorium-browser/thorium (PID 1613592)", + "result": "pass", + "evidence": "'exit code: 139' / 'died on signal: 11 (SEGV)' written to the real log. cmdline confirmed the 2026-07-25 mitigation flags AND --enable-logging=stderr --v=1 were both applied." + }, + { + "command": "pacman -S librewolf (previously blocked) after whitelist entry + install_pacman_wrapper.sh reinstall", + "result": "pass", + "evidence": "install succeeded; policy integrity checksums regenerated by the installer rather than edited in place. LeechBlock present as force_installed and uBlock Origin as normal_installed in the loaded profile (avpem8cg.default-default, per profiles.ini Locked=1); both active=True." + } + ], + "risks": [ + "The five auto-generated app-mode .desktop files under ~/.local/share/applications were rewritten from Exec=/opt/thorium-browser/thorium-browser to the wrapper. Chromium regenerates these, so re-creating an app shortcut silently reverts them — the same gap that left the 2026-07-25 flags un-applied. Not yet automated in setup_periodic_system.sh.", + "--v=1 is verbose; the watchdog bounds a single log but 20 retained logs can still reach ~640 MB worst case. Drop --v=1 if that proves too costly — the signal + VRAM discriminator does not depend on it.", + "The watchdog trims with tail -c, which can cut mid-line at the trim boundary.", + "Whitelisting librewolf widens the digital-wellbeing blocklist by exactly one package; enforcement depends on the LibreWolf policies.json remaining in place." + ], + "rollback": [ + "git revert the commit, then re-run install_pacman_wrapper.sh (regenerates policy checksums) and re-deploy the wrapper via the same sed substitution to /usr/local/bin/browser-preexec-wrapper.", + "Restore the .desktop files from the *.bak-precrashlog copies left alongside them.", + "After rollback, validate: `pacman -S --print librewolf` is blocked again, and a Thorium launch no longer creates files in ~/.local/state/browser-crash-logs." + ] +} diff --git a/linux_configuration/scripts/periodic_background/digital_wellbeing/pacman/pacman_whitelist.txt b/linux_configuration/scripts/periodic_background/digital_wellbeing/pacman/pacman_whitelist.txt index 425693f8..4662fd25 100644 --- a/linux_configuration/scripts/periodic_background/digital_wellbeing/pacman/pacman_whitelist.txt +++ b/linux_configuration/scripts/periodic_background/digital_wellbeing/pacman/pacman_whitelist.txt @@ -1,5 +1,12 @@ # Exact package names that should bypass the block even if matching a keyword. thorium-browser-bin +# Allowed 2026-08-09 as a LeechBlock-enforced browser, same footing as Thorium: +# install_leechblock.sh already policy-installs the blocker into LibreWolf via +# /etc/librewolf/policies, so this is not an unfiltered escape hatch. Needed +# because Chromium's MV2 sunset ends full uBlock Origin (with all filter lists) +# on Thorium; the Gecko build keeps it. The `librewolf` KEYWORD stays blocked, +# so every other Firefox fork remains barred. +librewolf minizip miniupnpc haskell-generically diff --git a/linux_configuration/scripts/periodic_background/setup_periodic_system.sh b/linux_configuration/scripts/periodic_background/setup_periodic_system.sh index 58d33961..ab757f24 100755 --- a/linux_configuration/scripts/periodic_background/setup_periodic_system.sh +++ b/linux_configuration/scripts/periodic_background/setup_periodic_system.sh @@ -204,6 +204,42 @@ install_browser_preexec_wrapper() { ln -sf "$wrapper" "$link" done echo "✓ Symlinked wrapper for common browsers in /usr/local/bin" + + redirect_app_mode_desktop_entries +} + +# Chromium "app mode" shortcuts (--app-id=...) are auto-generated with a direct +# Exec=/opt// line, bypassing /usr/local/bin entirely. That +# means neither the flags file nor the crash logging applies to them, and it is +# silent: on 2026-08-09 five such Thorium entries were found still launching +# without the --disable-gpu-compositing mitigation added on 2026-07-25. +# Chromium regenerates these files, so this re-runs on every install. +redirect_app_mode_desktop_entries() { + local target_user="${SUDO_USER:-$USER}" + local apps_dir + apps_dir="$(getent passwd "$target_user" | cut -d: -f6)/.local/share/applications" + + [[ -d $apps_dir ]] || return 0 + + local rewritten=0 f + while IFS= read -r -d '' f; do + if grep -q '^Exec=/opt/' "$f"; then + # Keep only the BASENAME, whatever the nesting depth: + # /opt/thorium-browser/thorium-browser -> /usr/local/bin/thorium-browser + # /opt/google/chrome/google-chrome -> /usr/local/bin/google-chrome + # The greedy .* up to the last slash is what makes depth irrelevant; + # matching a single path segment silently mangles the nested case. + sed -i -E 's|^Exec=/opt/.*/([^/ ]+)|Exec=/usr/local/bin/\1|' "$f" + rewritten=$((rewritten + 1)) + fi + done < <(find "$apps_dir" -maxdepth 1 -type f -name '*.desktop' -print0 2> /dev/null) + + if ((rewritten > 0)); then + chown "$target_user" "$apps_dir"/*.desktop 2> /dev/null || true + echo "✓ Redirected $rewritten app-mode .desktop entries through the wrapper" + else + echo "✓ No app-mode .desktop entries needed redirecting" + fi } # Function to install automatic system update service diff --git a/linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh b/linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh index bc17de78..f3c79ad9 100755 --- a/linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh +++ b/linux_configuration/scripts/periodic_background/system-maintenance/bin/browser-preexec-wrapper.sh @@ -1,33 +1,191 @@ #!/bin/bash # Generic pre-exec wrapper for browsers: ensures /etc/hosts is (re)installed -# before launching the actual browser binary. +# before launching the actual browser binary, and — for Chromium-family +# browsers — captures a crash log for every launch. +# +# Why the crash logging exists (diagnosed 2026-08-09): +# Thorium was "closing and dying randomly" with NO evidence anywhere: +# Crashpad was never enabled (no crash_reporting consent key, empty crash DB), +# and stderr from a dmenu/.desktop launch goes to no terminal and no journal. +# The only Thorium lines ever seen in the journal came from a systemd-launched +# instance. So the browser could die repeatedly and leave nothing behind. +# +# Each launch now gets its own log holding: a VRAM/RAM snapshot before start, +# the browser's own stderr (--enable-logging=stderr --v=1), and — on exit — +# the exit code, decoded signal, and a SECOND VRAM snapshot. That last pair is +# the discriminator between the two live hypotheses: a GPU/VRAM death shows +# NVRM/NV_ERR_NO_MEMORY or near-full VRAM at exit, whereas a stale-binary +# death shows SIGSEGV/SIGILL with the GPU idle. set -euo pipefail HOSTS_INSTALL_SCRIPT="__HOSTS_INSTALL_SCRIPT__" +# Where per-launch logs go, and how many to keep. +readonly LOG_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/browser-crash-logs" +readonly LOG_KEEP=20 + +# Per-log byte cap. Measured 2026-08-09: Thorium writes ~2 MB in the first 45s +# (65% of it VERBOSE1 net/TLS chatter) and ~0 while idle, so cost scales with +# navigation, not wall-clock. Capping COUNT alone is therefore not enough — a +# heavy browsing day could fill the disk, and a full disk causes exactly the +# kind of random death this logging exists to diagnose. +# +# Enforced by the watchdog below, which keeps the NEWEST bytes: the lines +# immediately before death are the diagnostic ones, so trimming must drop the +# head, never the tail. +readonly LOG_MAX_BYTES=$((32 * 1024 * 1024)) + prog_name="$(basename "$0")" real_bin="/usr/bin/${prog_name}" # If run directly (not via a browser symlink) or if the target binary doesn't exist, # allow passing the real browser command as the first argument for testing: if [[ ! -x $real_bin || $prog_name == "browser-preexec-wrapper.sh" ]]; then - if [[ $# -ge 1 ]]; then - real_bin="$1" - shift - else - echo "Error: could not resolve real browser binary for '$prog_name'." >&2 - echo "Usage (testing): $0 [args...]" >&2 - echo "Typical install: symlink this script as /usr/local/bin/ so it wraps /usr/bin/." >&2 - exit 127 - fi + if [[ $# -ge 1 ]]; then + real_bin="$1" + shift + else + echo "Error: could not resolve real browser binary for '$prog_name'." >&2 + echo "Usage (testing): $0 [args...]" >&2 + echo "Typical install: symlink this script as /usr/local/bin/ so it wraps /usr/bin/." >&2 + exit 127 + fi fi # Best-effort: install hosts file quietly; don't block browser startup -if command -v sudo > /dev/null 2>&1; then - sudo -n "$HOSTS_INSTALL_SCRIPT" > /dev/null 2>&1 || true +if command -v sudo >/dev/null 2>&1; then + sudo -n "$HOSTS_INSTALL_SCRIPT" >/dev/null 2>&1 || true else - "$HOSTS_INSTALL_SCRIPT" > /dev/null 2>&1 || true + "$HOSTS_INSTALL_SCRIPT" >/dev/null 2>&1 || true fi -exec "$real_bin" "$@" +# Only Chromium-family browsers understand --enable-logging/--v; everything else +# (firefox, librewolf) is exec'd unchanged so this wrapper stays generic. +is_chromium_family() { + case "$prog_name" in + thorium-browser | google-chrome | google-chrome-stable | chromium | brave | brave-browser | vivaldi-stable) + return 0 + ;; + *) return 1 ;; + esac +} + +# GPU + RAM state. Written before launch and again at exit so the two can be +# compared; nvidia-smi may be absent (or the GPU wedged), so never fail here. +emit_resource_snapshot() { + local label="$1" + echo "--- ${label} $(date '+%Y-%m-%d %H:%M:%S') ---" + if command -v nvidia-smi >/dev/null 2>&1; then + echo -n "VRAM used/total: " + nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader 2>&1 || echo "nvidia-smi failed" + else + echo "VRAM: nvidia-smi unavailable" + fi + echo -n "RAM: " + free -m | awk '/^Mem:/ {printf "%s MiB used / %s MiB total\n", $3, $2}' + echo "kernel: $(uname -r)" + # Match the dotted version anywhere on the line rather than by field index: + # the surrounding wording differs between open/proprietary driver builds. + # Worth logging because a module/userspace version skew (nvidia-utils upgraded + # without a reboot) is a known cause of GL crashes in Chromium. + if [[ -r /proc/driver/nvidia/version ]]; then + echo -n "nvidia module: " + grep -oE '[0-9]+\.[0-9]+\.[0-9]+' /proc/driver/nvidia/version 2>/dev/null | head -1 || echo "unreadable" + fi +} + +# Keep only the newest $keep logs. Uses -printf/sort -z so filenames containing +# spaces or newlines can't split a record. Called with LOG_KEEP-1 before a new +# log is created, so the directory settles at exactly LOG_KEEP afterwards. +rotate_logs() { + local keep="$1" + local old + while IFS= read -r -d '' old; do + rm -f -- "$old" + done < <( + find "$LOG_DIR" -maxdepth 1 -type f -name '*.log' -printf '%T@\t%p\0' 2>/dev/null | + sort -zrn | + tail -zn "+$((keep + 1))" | + cut -zf2- + ) +} + +if ! is_chromium_family; then + exec "$real_bin" "$@" +fi + +mkdir -p "$LOG_DIR" +rotate_logs "$((LOG_KEEP - 1))" +log_file="${LOG_DIR}/${prog_name}-$(date +%Y%m%d-%H%M%S)-$$.log" + +{ + echo "=== launch $(date '+%Y-%m-%d %H:%M:%S') ===" + echo "binary: $real_bin" + echo "args: $*" + emit_resource_snapshot "PRE-LAUNCH" + echo "=== browser stderr follows ===" +} >"$log_file" + +# Written from a trap rather than inline: if the wrapper itself is killed (the +# session ends, the process group is torn down, the OOM killer picks it), an +# inline block after `wait` would never run and the log would end mid-stream +# with no verdict — indistinguishable from a browser still running. +record_exit() { + local code="$1" + # Fire once; EXIT still runs after a signal trap has already reported. + [[ -n ${exit_recorded:-} ]] && return 0 + exit_recorded=1 + { + echo "=== exit ===" + echo "exit code: $code" + if ((code > 128)); then + local signal_num=$((code - 128)) + echo "died on signal: ${signal_num} ($(kill -l "$signal_num" 2>/dev/null || echo unknown))" + fi + emit_resource_snapshot "POST-EXIT" + } >>"$log_file" +} + +# Run in the background rather than exec'ing, so this shell survives to record +# how the browser died. `wait` reports 128+N for a signal death, which is the +# single most useful fact the old setup threw away. +# +# Deliberately a direct redirect, NOT a pipe into a size-limiting filter: in a +# pipeline `wait` reports the LAST element's status, which would report the +# filter's exit and discard the browser's signal — the one fact this whole +# wrapper exists to capture. Size is bounded by the watchdog below instead. +"$real_bin" --enable-logging=stderr --v=1 "$@" >>"$log_file" 2>&1 & +browser_pid=$! + +# Watchdog: if a browsing session makes the log exceed the cap, keep the most +# recent half and continue appending. Checks once a minute — the measured burst +# rate (~2 MB/45s of navigation, ~0 idle) can't overshoot 32 MB meaningfully in +# that window, and a 1/min poll costs nothing next to the browser itself. +( + while kill -0 "$browser_pid" 2>/dev/null; do + sleep 60 + size="$(stat -c %s "$log_file" 2>/dev/null || echo 0)" + if ((size > LOG_MAX_BYTES)); then + tail -c "$((LOG_MAX_BYTES / 2))" "$log_file" >"${log_file}.trim" 2>/dev/null && + mv "${log_file}.trim" "$log_file" && + echo "=== log trimmed to newest $((LOG_MAX_BYTES / 2)) bytes at $(date '+%H:%M:%S') ===" >>"$log_file" + fi + done +) & +watchdog_pid=$! + +# A signal aimed at the wrapper is recorded as the wrapper's own death, and the +# browser is taken down with it so no orphan outlives the log describing it. +# Written inline (not via a helper) so shellcheck can see the calls; each then +# re-raises with the default handler so the exit status stays truthful. +trap 'record_exit 129; kill "$browser_pid" "$watchdog_pid" 2>/dev/null || true; trap - HUP; kill -s HUP $$' HUP +trap 'record_exit 130; kill "$browser_pid" "$watchdog_pid" 2>/dev/null || true; trap - INT; kill -s INT $$' INT +trap 'record_exit 143; kill "$browser_pid" "$watchdog_pid" 2>/dev/null || true; trap - TERM; kill -s TERM $$' TERM + +exit_code=0 +wait "$browser_pid" || exit_code=$? + +kill "$watchdog_pid" 2>/dev/null || true +record_exit "$exit_code" +exit "$exit_code"