diff --git a/docs/superpowers/contracts/linux-config-searxng-split-2026-08-16.json b/docs/superpowers/contracts/linux-config-searxng-split-2026-08-16.json new file mode 100644 index 00000000..f4f54184 --- /dev/null +++ b/docs/superpowers/contracts/linux-config-searxng-split-2026-08-16.json @@ -0,0 +1,19 @@ +{ + "title": "Split setup_searxng.sh under the 250-line cap", + "objective": "Bring linux_configuration/scripts/single_use/features/setup_searxng.sh (912 lines) under the cap, and establish the verification a split must pass when the moved code generates service configuration.", + "acceptance_criteria": [ + "The entry script and all six new libs are at most 250 lines.", + "The pre-commit shellcheck hook passes on every touched file, with no new suppression and no disable directive.", + "No lib sources a sibling: the entry script sources all six directly.", + "The stubbed run reports every moved function as defined.", + "The content the script generates is byte-identical: comparing the non-comment lines of the original write_stack against the three functions it became must produce the same hash. A split that quietly altered a compose file or settings.yml would pass every other check.", + "The comment explaining why settings.yml is mode 640 rather than 600 travels with the code it explains.", + "No script is executed." + ], + "out_of_scope": [ + "Executing the setup, starting containers or reloading Caddy.", + "Changing anything the script writes.", + "The other very large scripts (setup_nextcloud_raspberry.sh, check_and_enable_services.sh and the rest), which are separate commits." + ], + "verifier": "meta/scripts/verify_shell_split.sh, plus an md5 comparison of the sorted non-comment lines of the original write_stack against its three successor functions, plus pre-commit run --files and check_file_length.sh." +} diff --git a/docs/superpowers/evidence/linux-config-searxng-split-2026-08-16.json b/docs/superpowers/evidence/linux-config-searxng-split-2026-08-16.json new file mode 100644 index 00000000..87804a0b --- /dev/null +++ b/docs/superpowers/evidence/linux-config-searxng-split-2026-08-16.json @@ -0,0 +1,43 @@ +{ + "intent": "Split setup_searxng.sh (912 lines) under the cap, the first of the very large scripts, and establish how to verify a split that moves generated config content.", + "scope": [ + "linux_configuration/scripts/single_use/features/setup_searxng.sh (912 -> 184)", + "lib/searxng_stack.sh, searxng_settings.sh, searxng_limiter.sh, searxng_start.sh, searxng_expose.sh, searxng_status.sh (new)", + "Non-goal: no script executed, no generated config content changed" + ], + "changes": [ + "Six libs rather than one or two: the script has four clear phases (write the stack, start it, expose it, report status) and write_stack alone was 342 lines because it emits three config files from heredocs.", + "write_stack became three functions, one per file written, keeping each heredoc with the comment that explains it - including the note about settings.yml needing mode 640 rather than 600, which records a real failure (the granian worker dies with EACCES at 600).", + "The entry script now sources all six libs directly; no lib sources a sibling." + ], + "verification": [ + { + "command": "meta/scripts/verify_shell_split.sh setup_searxng.sh write_stack write_searx_settings write_searx_limiter wait_for_site status_cmd setup_cmd ensure_firewall", + "result": "pass", + "evidence": "'libs sourced, functions defined' - all seven functions resolve across the six libs." + }, + { + "command": "compare the non-comment lines of the original write_stack against the three new functions, sorted, by md5", + "result": "pass", + "evidence": "c90298c6533a64458e74e3032b5d5279 on both sides. Every line of emitted compose, settings.yml and limiter.toml content is byte-identical; the only additions are the two function wrappers and the two calls to them. This is the check that matters for a script whose whole job is writing config - the stubbed run proves the functions exist, not that what they write is unchanged." + }, + { + "command": "pre-commit run --files ", + "result": "pass", + "evidence": "shellcheck, shfmt, the shebang-executable hook and jscpd all pass, with six new libs in one commit." + }, + { + "command": "bash ~/utils/scripts/check_file_length.sh --all", + "result": "pass", + "evidence": "None of the seven files is listed; the largest lib is 235 lines." + } + ], + "risks": [ + "Not executed: it writes /etc/searxng config, starts containers and reloads Caddy. The content-hash comparison is the substitute for running it, and is stronger evidence than the stubbed run for this particular script.", + "searxng_settings.sh is 235 lines, mostly one heredoc. Any addition to settings.yml pushes it over the cap, and the next seam there is not obvious - the heredoc cannot be split without changing what is written." + ], + "rollback": [ + "git revert the commit; setup_searxng.sh returns to 912 lines and the six libs disappear.", + "After rollback, re-run bash -n and shellcheck on the restored script." + ] +} diff --git a/linux_configuration/scripts/single_use/features/lib/searxng_expose.sh b/linux_configuration/scripts/single_use/features/lib/searxng_expose.sh new file mode 100755 index 00000000..64572358 --- /dev/null +++ b/linux_configuration/scripts/single_use/features/lib/searxng_expose.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Caddy snippet, firewall rules and the setup report. +# +# Sourced by setup_searxng.sh; split out to keep it under the 250-line +# cap. Sourced rather than run, so it inherits the caller's strict mode +# and the variables defined above the source line. + +# --- Phase 5: front it on the public domain --------------------------------- +# Header syntax matters and is easy to get wrong: +# Header "value" -> "set" (REPLACES any existing value) <-- what we want +# +Header "value" -> "add" (appends, yielding duplicates) +# >Header "value" -> "replace" (substring search/replace WITHIN a value — +# NOT a header setter. Using it here silently +# produced no header at all: it adapts to +# {"replace": {"": [{"search_regexp": "..."}]}} +# and matches nothing. Verified via caddy adapt.) +# This matters because SearXNG sends X-Content-Type-Options and Referrer-Policy +# itself via default_http_headers (its Referrer-Policy is `no-referrer`, which +# differs from ours), so the bare/"set" form is required to override rather than +# duplicate them. +write_snippet() { + cat >"$SEARX_SNIPPET" <' (substring-replace, which sets nothing). + Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' data:; font-src 'self'; connect-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'; manifest-src 'self'" + Strict-Transport-Security "max-age=31536000; includeSubDomains" + X-Content-Type-Options "nosniff" + Referrer-Policy "strict-origin-when-cross-origin" + Permissions-Policy "geolocation=(), microphone=(), camera=(), interest-cohort=()" + # No HSTS preload: duckdns.org is a public suffix, so kuhy.duckdns.org is + # the registrable domain — preloading would force HTTPS on every sibling + # subdomain and is effectively irreversible. Not needed to clear A+. + -Server + } + reverse_proxy 127.0.0.1:${SEARX_PORT} { + # SearXNG's botdetection needs the real client IP. Without this it logs + # "X-Forwarded-For nor X-Real-IP header is set!" and 429s every request, + # including the front page. Paired with trusted_proxies in limiter.toml — + # both halves are required, neither works alone. + header_up X-Forwarded-For {http.request.remote.host} + header_up X-Real-IP {http.request.remote.host} + } +} +EOF + log_ok "Wrote ${SEARX_SNIPPET}." +} + +# --- Phase 6: reload the edge ----------------------------------------------- +reload_caddy() { + if docker exec "$CADDY_CONTAINER" caddy validate --config /etc/caddy/Caddyfile >/dev/null 2>&1; then + docker exec "$CADDY_CONTAINER" caddy reload --config /etc/caddy/Caddyfile + log_ok "Reloaded ${CADDY_CONTAINER}." + else + rm -f "$SEARX_SNIPPET" + die "Caddy config invalid — snippet removed, edge left untouched." + fi +} + +# --- Phase 7: firewall ------------------------------------------------------ +# setup_wireguard_ssh.sh owns /etc/nftables.conf and regenerates it from scratch +# — `flush ruleset` and all — on every run, momentarily dropping the whole +# ruleset. Since all this needs is 80/443 already being accepted, check first +# and only pay for the regeneration when the rule is genuinely missing. +web_ports_open() { + local line ports + while read -r line; do + # Exact token membership: a loose regex would also match "8080, 4433". + ports=",${line//[^0-9,]/}," + if [[ $ports == *,80,* && $ports == *,443,* ]]; then + return 0 + fi + done < <(sudo -n nft list ruleset 2>/dev/null | grep -oE 'tcp dport \{[^}]*\}') + return 1 +} + +ensure_firewall() { + if web_ports_open; then + log_ok "Ports 80/443 already accepted — leaving the ruleset untouched." + return 0 + fi + log_info "Opening ports 80/443 via setup_wireguard_ssh.sh…" + sudo bash "$WG_SCRIPT" allow-web +} + +# --- Phase 8: report -------------------------------------------------------- +print_report() { + cat <"$SEARX_LIMITER" <<'EOF' +# Managed by setup_searxng.sh — do not edit by hand. +[botdetection] + +ipv4_prefix = 32 +ipv6_prefix = 48 + +# The edge (gitea-caddy) shares the host network namespace, so its requests +# arrive from loopback. Without 127.0.0.0/8 here the X-Forwarded-For that Caddy +# sends is ignored, every request looks like it came from the proxy itself, and +# rate limiting collapses onto a single bucket. +trusted_proxies = [ + '127.0.0.0/8', + '::1', +] + +[botdetection.ip_limit] +filter_link_local = false +# Off (the shipped default). link_token sends a probe the client must fetch to +# prove it renders HTML, which no scripted JSON client can satisfy. IP-based +# rate limiting below is unaffected and still applies. +link_token = false + +[botdetection.ip_lists] +block_ip = [] + +# pass_ip bypasses ALL botdetection (including the Sec-Fetch/User-Agent probe +# headers) for these addresses. +# +# This is what makes search.formats = [html, json] usable. A scripted JSON +# client — an MCP server, a script — cannot send Sec-Fetch-Mode/Site/Dest, and +# botdetection 429s anything that lacks them, so WITHOUT this entry the JSON API +# is unreachable for every non-browser client. Verified: with browser headers a +# search returns 200 + results; the identical request without Sec-Fetch-* gets +# 429. +# +# Deliberately LOOPBACK ONLY. Requests arriving through the public edge carry +# the real client IP in X-Forwarded-For (trusted_proxies above), so they are +# still fully rate-limited — this exempts only tools running on this host. +pass_ip = [ + '127.0.0.0/8', + '::1', +] +EOF + chmod 640 "$SEARX_LIMITER" + log_ok "Wrote ${SEARX_LIMITER}." +} diff --git a/linux_configuration/scripts/single_use/features/lib/searxng_settings.sh b/linux_configuration/scripts/single_use/features/lib/searxng_settings.sh new file mode 100755 index 00000000..296045ea --- /dev/null +++ b/linux_configuration/scripts/single_use/features/lib/searxng_settings.sh @@ -0,0 +1,235 @@ +#!/bin/bash +# The SearXNG settings.yml, including why it is mode 640 and not 600. +# +# Sourced by setup_searxng.sh; split out to keep searxng_stack.sh under +# the 250-line cap. Sourced rather than run, so it inherits the caller's +# strict mode and the variables defined above the source line. + +write_searx_settings() { + local secret + secret="$(read_or_make_secret)" + + cat >"$SEARX_SETTINGS" < 20 results, !bi -> 10, both +# verified working). So the pruning removed only engines that were already +# contributing zero results -- but it also means general search has a single +# point of failure. If google cse ever starts erroring, general search returns +# nothing until another engine is enabled or !go/!bi is used. +# +# pinterest is blackholed to 0.0.0.0 by this host's own /etc/hosts +# (generate_hosts_file.sh) -- a LOCAL failure that looks identical to a dead +# upstream. Disabled to stop the pointless request, not because the engine is bad. +# +# USE "inactive: true", NOT "disabled: true". Straight from enginelib: +# disabled -> "disable BY DEFAULT ... will allow the user to manually +# activate it in the settings" (i.e. only the default for a +# fresh visitor; anyone with an existing preferences cookie keeps +# running the engine) +# inactive -> "Remove the engine from the settings (disabled & removed)" +# The first pass used "disabled" and the pruned engines kept executing for any +# browser session that already had a preferences cookie -- yandex images was +# still throwing JSONDecodeErrors in the log minutes after being "disabled". +# +# EXCEPTION -- brave and qwant use "disabled", not "inactive", ON PURPOSE. +# They own an outgoing connection pool that sibling engines reference by name +# ("network: brave" <- brave.images/.videos/.news; "network: qwant" <- qwant +# news/images/videos). "inactive" REMOVES the definition, so the siblings' +# lookup dies at startup with KeyError: 'qwant' and the container never comes +# up. "disabled" keeps the pool defined while hiding the engine from results. +# Pool owners in this image: brave, qwant, yandex, piped, yacy -- of which only +# brave and qwant are pruned here. Check before making any of the rest inactive: +# grep -B4 'network: ' /usr/local/searxng/searx/settings.yml +# +# Re-check periodically: +# curl -s 'http://127.0.0.1:8090/search?q=test&format=json' | grep -o '"unresponsive_engines":.*' +engines: + # === general === + - { name: brave, disabled: true } # too many requests + - { name: duckduckgo, inactive: true } # CAPTCHA + - { name: startpage, inactive: true } # Suspended: CAPTCHA + - { name: dogpile, inactive: true } # access denied + - { name: encyclosearch, inactive: true } # timeout + - { name: fastbot, inactive: true } # access denied + - { name: fireball, inactive: true } # access denied + - { name: quark, inactive: true } # unexpected crash + - { name: qwant, disabled: true } # CAPTCHA + - { name: sogou, inactive: true } # CAPTCHA + - { name: tusksearch, inactive: true } # HTTP error + - { name: yahoo, inactive: true } # HTTP protocol error + - { name: yep, inactive: true } # access denied + - { name: naver, inactive: true } # 1.9s + - { name: baidu, inactive: true } # 1.7s + - { name: abcnyheter, inactive: true } # 1.5s + - { name: boardreader, inactive: true } # 1.5s + - { name: crowdview, inactive: true } # 1.0s + + # === images === (keeping ONLY google cse images) + - { name: dogpile images, inactive: true } # access denied + - { name: findfiles images, inactive: true } # unexpected crash + - { name: library of congress, inactive: true } # parsing error + - { name: pinterest, inactive: true } # 0.0.0.0 in /etc/hosts (local) + - { name: qwant images, inactive: true } # CAPTCHA + - { name: tusksearch images, inactive: true } # HTTP error + - { name: wikicommons.images, inactive: true } # too many requests + - { name: yandex images, inactive: true } # parsing error + - { name: quark images, inactive: true } # 2.9s + - { name: 1x, inactive: true } # 2.5s + - { name: baidu images, inactive: true } # 2.5s + - { name: picjumbo, inactive: true } # 2.4s + - { name: naver images, inactive: true } # 2.3s + - { name: sogou images, inactive: true } # 2.0s + - { name: unsplash, inactive: true } # 1.2s + - { name: duckduckgo images, inactive: true } + - { name: flickr, inactive: true } # 1.0s + - { name: bing images, inactive: true } # cut for the sub-1s images goal + - { name: deviantart, inactive: true } # cut for the sub-1s images goal + - { name: openverse, inactive: true } # cut for the sub-1s images goal + - { name: pexels, inactive: true } # cut for the sub-1s images goal + - { name: artic, inactive: true } # cut for the sub-1s images goal + - { name: devicons, inactive: true } # cut for the sub-1s images goal + - { name: lucide, inactive: true } # cut for the sub-1s images goal + - { name: wallhaven, inactive: true } # cut for the sub-1s images goal + # Not in the /preferences list kuhy sent -- these only surfaced as + # unresponsive under load, same failure pattern, so pruned with the rest. + - { name: brave.images, inactive: true } # too many requests + - { name: startpage images, inactive: true } # Suspended: CAPTCHA + + # === videos === + - { name: 360search videos, inactive: true } # unexpected crash + - { name: acfun, inactive: true } # timeout + - { name: brave.videos, inactive: true } # too many requests + - { name: dogpile videos, inactive: true } # access denied + - { name: fireball videos, inactive: true } # access denied + - { name: niconico, inactive: true } # HTTP connection error + - { name: pixabay videos, inactive: true } # timeout + - { name: tusksearch videos, inactive: true } # HTTP error + - { name: vimeo, inactive: true } # access denied + - { name: youtube, inactive: true } # HTTP connection error + - { name: naver videos, inactive: true } # 2.4s + - { name: sogou videos, inactive: true } # 2.0s + - { name: iqiyi, inactive: true } # 1.3s + - { name: rumble, inactive: true } # 1.3s + + # === news === + - { name: bing news, inactive: true } # parsing error + - { name: brave.news, inactive: true } # too many requests + - { name: dogpile news, inactive: true } # access denied + - { name: fireball news, inactive: true } # access denied + - { name: google news, inactive: true } # Suspended: CAPTCHA + - { name: startpage news, inactive: true } # Suspended: CAPTCHA + - { name: tusksearch news, inactive: true } # HTTP error + - { name: sogou wechat, inactive: true } # 1.7s + - { name: naver news, inactive: true } # 1.7s + + # === music === + - { name: radio browser, inactive: true } # HTTP error + - { name: yandex music, inactive: true } # HTTP error + + # === it === + - { name: codeberg, inactive: true } # timeout + - { name: metacpan, inactive: true } # HTTP error + - { name: nixos wiki, inactive: true } # timeout + - { name: baidu kaifa, inactive: true } # 2.0s + - { name: gitea.com, inactive: true } # 1.8s + - { name: rubygems, inactive: true } # 1.1s + + # === science === + - { name: openairepublications, inactive: true } # 2.7s + - { name: openairedatasets, inactive: true } # 2.7s + + # === files === + - { name: 1337x, inactive: true } # HTTP connection error + - { name: btdigg, inactive: true } # 1.5s + - { name: findfiles, inactive: true } # 1.2s + - { name: kickass, inactive: true } # HTTP connection error (surfaced under load) + + # === social media === + - { name: 9gag, inactive: true } # access denied + - { name: tootfinder, inactive: true } # access denied +EOF + # 640, NOT 600. The entrypoint chowns this file to the container's searxng + # user (uid 977), but granian's worker runs as a different uid and only has + # GROUP access — mode 600 makes it unreadable and the worker dies with + # "[Errno 13] Permission denied: /etc/searxng/settings.yml". It still is not + # world-readable, so secret_key stays protected. + chmod 640 "$SEARX_SETTINGS" + log_ok "Wrote ${SEARX_SETTINGS}." +} diff --git a/linux_configuration/scripts/single_use/features/lib/searxng_stack.sh b/linux_configuration/scripts/single_use/features/lib/searxng_stack.sh new file mode 100755 index 00000000..b28688ed --- /dev/null +++ b/linux_configuration/scripts/single_use/features/lib/searxng_stack.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# The SearXNG compose stack and settings files. +# +# Sourced by setup_searxng.sh; split out to keep it under the 250-line +# cap. Sourced rather than run, so it inherits the caller's strict mode +# and the variables defined above the source line. + +write_stack() { + ensure_dir "$SEARX_CONFIG_DIR" + reclaim_config_ownership + + cat >"$SEARX_COMPOSE" </dev/null || echo 0)" + if [[ ${size:-0} -gt 0 ]]; then + return 0 + fi + sleep 1 + done + return 1 +} + +wait_for_valkey() { + local _ out + for _ in $(seq 1 30); do + out="$(docker exec "$VALKEY_CONTAINER" valkey-cli -h 127.0.0.1 -p "$VALKEY_PORT" ping 2>/dev/null || true)" + if [[ $out == "PONG" ]]; then + return 0 + fi + sleep 1 + done + return 1 +} + +# These containers share the host network namespace with the public edge, so a +# misconfiguration does not fail locally — it quietly exposes an unfronted +# service to the whole LAN, or steals a share of :80/:443 from every other site +# on the box. Gate on it rather than trust the config to stay correct. +assert_loopback_only() { + local container="$1" port="$2" pid listeners + pid="$(docker inspect -f '{{.State.Pid}}' "$container" 2>/dev/null || echo "")" + [[ -n $pid && $pid != "0" ]] || die "Could not read the ${container} PID." + + listeners="$(sudo -n ss -ltnp 2>/dev/null | grep "pid=${pid}," || true)" + + # Demand positive evidence FIRST. Asserting only the absence of a bad bind + # fails open: if `ss` returns nothing — a stale PID, or sudo -n unavailable — + # every "is it bad?" test misses and the guard reports success having looked + # at nothing. + if ! grep -qE '(^|[[:space:]])127\.0\.0\.1:'"${port}"'([[:space:]]|$)' <<<"$listeners"; then + docker stop "$container" >/dev/null 2>&1 || true + die "Could not confirm ${container} listens on 127.0.0.1:${port}. Container stopped." + fi + if grep -qE ':(80|443)\b' <<<"$listeners"; then + docker stop "$container" >/dev/null 2>&1 || true + die "${container} bound :80/:443 — it would contend with ${CADDY_CONTAINER}. Container stopped." + fi + # Reject *, 0.0.0.0 AND [::] — granian's real default is ::, which a + # 0.0.0.0-only check would sail straight past. + if grep -qE '(\*|0\.0\.0\.0|\[::\]):'"${port}"'\b' <<<"$listeners"; then + docker stop "$container" >/dev/null 2>&1 || true + die "${container} bound ${port} on all interfaces, not loopback. Container stopped." + fi + log_ok "${container} listens on loopback:${port} only — no contention, not LAN-exposed." +} + +start_stack() { + log_info "Starting valkey + searxng…" + docker compose -f "$SEARX_COMPOSE" up -d + # `up -d` is a no-op when only a BIND-MOUNTED config file changed — compose + # compares the compose spec, not the files it mounts. settings.yml and + # limiter.toml are read once at startup, so without an explicit restart the + # container keeps serving the previous config and every verification below + # silently tests the OLD deployment. (Hit for real: limiter.toml was written + # at 15:06 while the container had been running since 15:02, so botdetection + # kept 429ing with "missing config file".) + log_info "Restarting searxng to pick up settings.yml / limiter.toml…" + docker restart "$SEARX_CONTAINER" >/dev/null + + if wait_for_valkey; then + log_ok "valkey answering PING on 127.0.0.1:${VALKEY_PORT}." + else + die "valkey did not become ready — the limiter would silently no-op." + fi + + if wait_for_site; then + log_ok "SearXNG answering on http://127.0.0.1:${SEARX_PORT}/." + else + docker stop "$SEARX_CONTAINER" >/dev/null 2>&1 || true + die "SearXNG did not become reachable on 127.0.0.1:${SEARX_PORT} (container stopped)." + fi + + assert_loopback_only "$VALKEY_CONTAINER" "$VALKEY_PORT" + assert_loopback_only "$SEARX_CONTAINER" "$SEARX_PORT" +} diff --git a/linux_configuration/scripts/single_use/features/lib/searxng_status.sh b/linux_configuration/scripts/single_use/features/lib/searxng_status.sh new file mode 100755 index 00000000..c8d1f652 --- /dev/null +++ b/linux_configuration/scripts/single_use/features/lib/searxng_status.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# The status subcommand and its HTTP, container and header checks. +# +# Sourced by setup_searxng.sh; split out to keep it under the 250-line +# cap. Sourced rather than run, so it inherits the caller's strict mode +# and the variables defined above the source line. + +# --- status: self-diagnose -------------------------------------------------- +status_line() { + if [[ $1 -eq 0 ]]; then + log_ok "$2" + else + log_warn "$2" + fi +} + +check_http() { + # Prints 0 to stdout if the URL returns 200 with a non-empty body, else 1. + # The body-size check matters: a host-matcher misconfiguration answers 200 + # with nothing in it, which a status-only check would call healthy. + # Certificates are NOT skipped — `curl -k` would report a failed-ACME cert as + # healthy, which is exactly the green-while-broken this exists to catch. + # Browser-shaped headers are REQUIRED, not cosmetic. botdetection runs a + # separate check per header and 429s on the FIRST one that fails, so the full + # set below must be sent every time. All four matter: + # Accept-Language -> http_accept_language (omitting this alone 429s) + # Accept-Encoding -> http_accept_encoding (needs gzip or deflate) + # Accept -> http_accept + # Sec-Fetch-* -> http_sec_fetch (302-redirects when invalid) + # Diagnosing which one bit is easy but non-obvious: set `debug: true` under + # general: in settings.yml, restart, and the log prints + # "NOT OK (searx.botdetection.)" naming the exact check. + # (Observed: dropping only Accept-Language turned a healthy 200 into a 429.) + local url="$1" out code size + out="$(curl -s --compressed -o /dev/null -w '%{http_code} %{size_download}' --max-time 10 \ + -A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Accept-Encoding: gzip, deflate, br' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-Dest: document' \ + ${2:+-H "Host: $2"} "$url" 2>/dev/null || echo "000 0")" + read -r code size <<<"$out" + [[ $code == "200" && ${size:-0} -gt 0 ]] && echo 0 || echo 1 +} + +check_container() { + local name="$1" port="$2" pid listeners + if ! docker ps --format '{{.Names}}' | grep -qx "$name"; then + status_line 1 "${name} container not running" + return + fi + status_line 0 "${name} container running" + pid="$(docker inspect -f '{{.State.Pid}}' "$name" 2>/dev/null || echo "")" + listeners="$(sudo -n ss -ltnp 2>/dev/null | grep "pid=${pid}," || true)" + if [[ -z $listeners ]]; then + status_line 1 "${name}: could not read listeners (needs sudo -n) — binding UNVERIFIED" + elif grep -qE '(\*|0\.0\.0\.0|\[::\]):'"${port}"'\b' <<<"$listeners"; then + status_line 1 "${name} is bound to ALL INTERFACES on ${port} — LAN-exposed" + elif grep -qE '(^|[[:space:]])127\.0\.0\.1:'"${port}"'([[:space:]]|$)' <<<"$listeners"; then + status_line 0 "${name} listens on loopback:${port} only" + else + status_line 1 "${name} not listening on ${port} as expected" + fi +} + +# Duplicated headers are the classic silent A+ killer, so count them rather than +# merely checking presence. +check_headers() { + local headers out h count + # Same browser-header requirement as check_http — a 429 carries none of these + # headers, so a bare curl would report all five as MISSING on a healthy site. + out="$(curl -sI --compressed --max-time 10 \ + -A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Accept-Encoding: gzip, deflate, br' \ + -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: none' -H 'Sec-Fetch-Dest: document' \ + "https://${SEARX_DOMAIN}/" 2>/dev/null || true)" + if [[ -z $out ]]; then + status_line 1 "could not fetch headers from https://${SEARX_DOMAIN}/" + return + fi + headers=(content-security-policy strict-transport-security + x-content-type-options referrer-policy permissions-policy) + for h in "${headers[@]}"; do + count="$(grep -ci "^${h}:" <<<"$out" || true)" + if [[ $count -eq 1 ]]; then + status_line 0 "header ${h}: exactly 1" + elif [[ $count -eq 0 ]]; then + status_line 1 "header ${h}: MISSING" + else + status_line 1 "header ${h}: DUPLICATED (${count}) — costs the A+ grade" + fi + done +} + +status_cmd() { + print_setup_header "SearXNG status" + + if has_cmd docker; then + status_line 0 "docker present" + else + status_line 1 "docker missing" + fi + + # NOTE: deliberately no ownership/mode assertion on settings.yml — the + # entrypoint chowns the bind mount to uid 977, so a "must be kuhy-owned, + # mode 600" check would false-fail on every run after first start. + if [[ -f $SEARX_SETTINGS ]]; then + status_line 0 "settings.yml present" + else + status_line 1 "settings.yml missing — run setup" + fi + # Its absence is not cosmetic: with no limiter.toml there are no + # trusted_proxies, so botdetection 429s every request including the homepage. + if [[ -f $SEARX_LIMITER ]]; then + status_line 0 "limiter.toml present" + else + status_line 1 "limiter.toml missing — botdetection will 429 everything" + fi + + check_container "$VALKEY_CONTAINER" "$VALKEY_PORT" + check_container "$SEARX_CONTAINER" "$SEARX_PORT" + + # The limiter silently no-ops when valkey is unreachable, so a working search + # does NOT prove the limiter works. Check the backend directly. + local ping + ping="$(docker exec "$VALKEY_CONTAINER" valkey-cli -h 127.0.0.1 -p "$VALKEY_PORT" ping 2>/dev/null || true)" + if [[ $ping == "PONG" ]]; then + status_line 0 "valkey reachable — limiter backend live" + else + status_line 1 "valkey NOT reachable — the limiter is silently disabled" + fi + + status_line "$(check_http "http://127.0.0.1:${SEARX_PORT}/" "$SEARX_DOMAIN")" \ + "local SearXNG (127.0.0.1:${SEARX_PORT}, Host: ${SEARX_DOMAIN})" + + if [[ -f $SEARX_SNIPPET ]]; then + status_line 0 "Caddy snippet present" + else + status_line 1 "Caddy snippet missing" + fi + if getent hosts "$SEARX_DOMAIN" >/dev/null; then + status_line 0 "${SEARX_DOMAIN} resolves" + else + status_line 1 "${SEARX_DOMAIN} does not resolve" + fi + status_line "$(check_http "https://${SEARX_DOMAIN}/")" \ + "external https://${SEARX_DOMAIN}/" + + check_headers + + # A loading front page proves nothing about the actual product. Egress is the + # fragile part on this host (bridge networking has no outbound access at all), + # so run a real query and require real results. Uses the loopback JSON API, + # which pass_ip exempts from botdetection. + local json hits + json="$(curl -s --compressed --max-time 30 \ + "http://127.0.0.1:${SEARX_PORT}/search?q=arch+linux&format=json" 2>/dev/null || true)" + hits="$(grep -o '"url"' <<<"$json" | wc -l)" + if [[ ${hits:-0} -gt 0 ]]; then + status_line 0 "live search returned ${hits} results (engines reachable)" + else + status_line 1 "live search returned NO results — check egress/engines" + fi + # Surfaces engines that are failing rather than merely slow; these are the + # candidates for pruning. + local dead + dead="$(grep -oE '"unresponsive_engines": \[\[[^]]*' <<<"$json" | head -c 300 || true)" + [[ -n $dead ]] && log_info "unresponsive engines: ${dead#*: }" + + # After pruning, general search is served by google cse alone (google/bing + # are inactive: true upstream and only answer to !go / !bi). That is a single + # point of failure worth naming explicitly rather than discovering as "no + # results", so check the engine that is actually carrying the category. + if [[ ${hits:-0} -eq 0 ]]; then + log_warn "general search has no working engine — try: !go / !bi ," + log_warn "or re-enable an engine in the 'engines:' block of ${SEARX_SETTINGS}" + fi + + log_info "Not checked here (needs another machine): the JSON API under bot" + log_info "detection. Run from your phone on cellular, not from this host:" + log_info " curl 'https://${SEARX_DOMAIN}/search?q=test&format=json'" +} diff --git a/linux_configuration/scripts/single_use/features/setup_searxng.sh b/linux_configuration/scripts/single_use/features/setup_searxng.sh index 3374cab6..367b98b0 100755 --- a/linux_configuration/scripts/single_use/features/setup_searxng.sh +++ b/linux_configuration/scripts/single_use/features/setup_searxng.sh @@ -148,747 +148,19 @@ reclaim_config_ownership() { done } -write_stack() { - ensure_dir "$SEARX_CONFIG_DIR" - reclaim_config_ownership +# shellcheck source=lib/searxng_stack.sh +source "$SCRIPT_DIR/lib/searxng_stack.sh" +# shellcheck source=lib/searxng_settings.sh +source "$SCRIPT_DIR/lib/searxng_settings.sh" +# shellcheck source=lib/searxng_limiter.sh +source "$SCRIPT_DIR/lib/searxng_limiter.sh" +# shellcheck source=lib/searxng_start.sh +source "$SCRIPT_DIR/lib/searxng_start.sh" +# shellcheck source=lib/searxng_expose.sh +source "$SCRIPT_DIR/lib/searxng_expose.sh" +# shellcheck source=lib/searxng_status.sh +source "$SCRIPT_DIR/lib/searxng_status.sh" - cat >"$SEARX_COMPOSE" <"$SEARX_SETTINGS" < 20 results, !bi -> 10, both -# verified working). So the pruning removed only engines that were already -# contributing zero results -- but it also means general search has a single -# point of failure. If google cse ever starts erroring, general search returns -# nothing until another engine is enabled or !go/!bi is used. -# -# pinterest is blackholed to 0.0.0.0 by this host's own /etc/hosts -# (generate_hosts_file.sh) -- a LOCAL failure that looks identical to a dead -# upstream. Disabled to stop the pointless request, not because the engine is bad. -# -# USE "inactive: true", NOT "disabled: true". Straight from enginelib: -# disabled -> "disable BY DEFAULT ... will allow the user to manually -# activate it in the settings" (i.e. only the default for a -# fresh visitor; anyone with an existing preferences cookie keeps -# running the engine) -# inactive -> "Remove the engine from the settings (disabled & removed)" -# The first pass used "disabled" and the pruned engines kept executing for any -# browser session that already had a preferences cookie -- yandex images was -# still throwing JSONDecodeErrors in the log minutes after being "disabled". -# -# EXCEPTION -- brave and qwant use "disabled", not "inactive", ON PURPOSE. -# They own an outgoing connection pool that sibling engines reference by name -# ("network: brave" <- brave.images/.videos/.news; "network: qwant" <- qwant -# news/images/videos). "inactive" REMOVES the definition, so the siblings' -# lookup dies at startup with KeyError: 'qwant' and the container never comes -# up. "disabled" keeps the pool defined while hiding the engine from results. -# Pool owners in this image: brave, qwant, yandex, piped, yacy -- of which only -# brave and qwant are pruned here. Check before making any of the rest inactive: -# grep -B4 'network: ' /usr/local/searxng/searx/settings.yml -# -# Re-check periodically: -# curl -s 'http://127.0.0.1:8090/search?q=test&format=json' | grep -o '"unresponsive_engines":.*' -engines: - # === general === - - { name: brave, disabled: true } # too many requests - - { name: duckduckgo, inactive: true } # CAPTCHA - - { name: startpage, inactive: true } # Suspended: CAPTCHA - - { name: dogpile, inactive: true } # access denied - - { name: encyclosearch, inactive: true } # timeout - - { name: fastbot, inactive: true } # access denied - - { name: fireball, inactive: true } # access denied - - { name: quark, inactive: true } # unexpected crash - - { name: qwant, disabled: true } # CAPTCHA - - { name: sogou, inactive: true } # CAPTCHA - - { name: tusksearch, inactive: true } # HTTP error - - { name: yahoo, inactive: true } # HTTP protocol error - - { name: yep, inactive: true } # access denied - - { name: naver, inactive: true } # 1.9s - - { name: baidu, inactive: true } # 1.7s - - { name: abcnyheter, inactive: true } # 1.5s - - { name: boardreader, inactive: true } # 1.5s - - { name: crowdview, inactive: true } # 1.0s - - # === images === (keeping ONLY google cse images) - - { name: dogpile images, inactive: true } # access denied - - { name: findfiles images, inactive: true } # unexpected crash - - { name: library of congress, inactive: true } # parsing error - - { name: pinterest, inactive: true } # 0.0.0.0 in /etc/hosts (local) - - { name: qwant images, inactive: true } # CAPTCHA - - { name: tusksearch images, inactive: true } # HTTP error - - { name: wikicommons.images, inactive: true } # too many requests - - { name: yandex images, inactive: true } # parsing error - - { name: quark images, inactive: true } # 2.9s - - { name: 1x, inactive: true } # 2.5s - - { name: baidu images, inactive: true } # 2.5s - - { name: picjumbo, inactive: true } # 2.4s - - { name: naver images, inactive: true } # 2.3s - - { name: sogou images, inactive: true } # 2.0s - - { name: unsplash, inactive: true } # 1.2s - - { name: duckduckgo images, inactive: true } - - { name: flickr, inactive: true } # 1.0s - - { name: bing images, inactive: true } # cut for the sub-1s images goal - - { name: deviantart, inactive: true } # cut for the sub-1s images goal - - { name: openverse, inactive: true } # cut for the sub-1s images goal - - { name: pexels, inactive: true } # cut for the sub-1s images goal - - { name: artic, inactive: true } # cut for the sub-1s images goal - - { name: devicons, inactive: true } # cut for the sub-1s images goal - - { name: lucide, inactive: true } # cut for the sub-1s images goal - - { name: wallhaven, inactive: true } # cut for the sub-1s images goal - # Not in the /preferences list kuhy sent -- these only surfaced as - # unresponsive under load, same failure pattern, so pruned with the rest. - - { name: brave.images, inactive: true } # too many requests - - { name: startpage images, inactive: true } # Suspended: CAPTCHA - - # === videos === - - { name: 360search videos, inactive: true } # unexpected crash - - { name: acfun, inactive: true } # timeout - - { name: brave.videos, inactive: true } # too many requests - - { name: dogpile videos, inactive: true } # access denied - - { name: fireball videos, inactive: true } # access denied - - { name: niconico, inactive: true } # HTTP connection error - - { name: pixabay videos, inactive: true } # timeout - - { name: tusksearch videos, inactive: true } # HTTP error - - { name: vimeo, inactive: true } # access denied - - { name: youtube, inactive: true } # HTTP connection error - - { name: naver videos, inactive: true } # 2.4s - - { name: sogou videos, inactive: true } # 2.0s - - { name: iqiyi, inactive: true } # 1.3s - - { name: rumble, inactive: true } # 1.3s - - # === news === - - { name: bing news, inactive: true } # parsing error - - { name: brave.news, inactive: true } # too many requests - - { name: dogpile news, inactive: true } # access denied - - { name: fireball news, inactive: true } # access denied - - { name: google news, inactive: true } # Suspended: CAPTCHA - - { name: startpage news, inactive: true } # Suspended: CAPTCHA - - { name: tusksearch news, inactive: true } # HTTP error - - { name: sogou wechat, inactive: true } # 1.7s - - { name: naver news, inactive: true } # 1.7s - - # === music === - - { name: radio browser, inactive: true } # HTTP error - - { name: yandex music, inactive: true } # HTTP error - - # === it === - - { name: codeberg, inactive: true } # timeout - - { name: metacpan, inactive: true } # HTTP error - - { name: nixos wiki, inactive: true } # timeout - - { name: baidu kaifa, inactive: true } # 2.0s - - { name: gitea.com, inactive: true } # 1.8s - - { name: rubygems, inactive: true } # 1.1s - - # === science === - - { name: openairepublications, inactive: true } # 2.7s - - { name: openairedatasets, inactive: true } # 2.7s - - # === files === - - { name: 1337x, inactive: true } # HTTP connection error - - { name: btdigg, inactive: true } # 1.5s - - { name: findfiles, inactive: true } # 1.2s - - { name: kickass, inactive: true } # HTTP connection error (surfaced under load) - - # === social media === - - { name: 9gag, inactive: true } # access denied - - { name: tootfinder, inactive: true } # access denied -EOF - # 640, NOT 600. The entrypoint chowns this file to the container's searxng - # user (uid 977), but granian's worker runs as a different uid and only has - # GROUP access — mode 600 makes it unreadable and the worker dies with - # "[Errno 13] Permission denied: /etc/searxng/settings.yml". It still is not - # world-readable, so secret_key stays protected. - chmod 640 "$SEARX_SETTINGS" - log_ok "Wrote ${SEARX_SETTINGS}." - - # Without this file SearXNG logs "missing config file: - # /etc/searxng/limiter.toml" and runs with no trusted_proxies, so the - # forwarded client IP is never trusted and botdetection 429s *everything*. - cat >"$SEARX_LIMITER" <<'EOF' -# Managed by setup_searxng.sh — do not edit by hand. -[botdetection] - -ipv4_prefix = 32 -ipv6_prefix = 48 - -# The edge (gitea-caddy) shares the host network namespace, so its requests -# arrive from loopback. Without 127.0.0.0/8 here the X-Forwarded-For that Caddy -# sends is ignored, every request looks like it came from the proxy itself, and -# rate limiting collapses onto a single bucket. -trusted_proxies = [ - '127.0.0.0/8', - '::1', -] - -[botdetection.ip_limit] -filter_link_local = false -# Off (the shipped default). link_token sends a probe the client must fetch to -# prove it renders HTML, which no scripted JSON client can satisfy. IP-based -# rate limiting below is unaffected and still applies. -link_token = false - -[botdetection.ip_lists] -block_ip = [] - -# pass_ip bypasses ALL botdetection (including the Sec-Fetch/User-Agent probe -# headers) for these addresses. -# -# This is what makes search.formats = [html, json] usable. A scripted JSON -# client — an MCP server, a script — cannot send Sec-Fetch-Mode/Site/Dest, and -# botdetection 429s anything that lacks them, so WITHOUT this entry the JSON API -# is unreachable for every non-browser client. Verified: with browser headers a -# search returns 200 + results; the identical request without Sec-Fetch-* gets -# 429. -# -# Deliberately LOOPBACK ONLY. Requests arriving through the public edge carry -# the real client IP in X-Forwarded-For (trusted_proxies above), so they are -# still fully rate-limited — this exempts only tools running on this host. -pass_ip = [ - '127.0.0.0/8', - '::1', -] -EOF - chmod 640 "$SEARX_LIMITER" - log_ok "Wrote ${SEARX_LIMITER}." -} - -# --- Phase 3/4: start ------------------------------------------------------- -# Sends the Host header the edge will actually forward, and requires a -# non-empty body. A bare "curl http://127.0.0.1:PORT/" against a host-matcher -# misconfiguration returns an empty 200 — it passes while every real request -# serves a blank page. -wait_for_site() { - local _ size - for _ in $(seq 1 60); do - # Browser-shaped headers: without them botdetection answers 429 and this - # readiness probe would spin for its full 60s against a healthy backend. - size="$(curl -s --compressed -o /dev/null -w '%{size_download}' \ - -A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: none' -H 'Sec-Fetch-Dest: document' \ - -H "Host: ${SEARX_DOMAIN}" "http://127.0.0.1:${SEARX_PORT}/" 2>/dev/null || echo 0)" - if [[ ${size:-0} -gt 0 ]]; then - return 0 - fi - sleep 1 - done - return 1 -} - -wait_for_valkey() { - local _ out - for _ in $(seq 1 30); do - out="$(docker exec "$VALKEY_CONTAINER" valkey-cli -h 127.0.0.1 -p "$VALKEY_PORT" ping 2>/dev/null || true)" - if [[ $out == "PONG" ]]; then - return 0 - fi - sleep 1 - done - return 1 -} - -# These containers share the host network namespace with the public edge, so a -# misconfiguration does not fail locally — it quietly exposes an unfronted -# service to the whole LAN, or steals a share of :80/:443 from every other site -# on the box. Gate on it rather than trust the config to stay correct. -assert_loopback_only() { - local container="$1" port="$2" pid listeners - pid="$(docker inspect -f '{{.State.Pid}}' "$container" 2>/dev/null || echo "")" - [[ -n $pid && $pid != "0" ]] || die "Could not read the ${container} PID." - - listeners="$(sudo -n ss -ltnp 2>/dev/null | grep "pid=${pid}," || true)" - - # Demand positive evidence FIRST. Asserting only the absence of a bad bind - # fails open: if `ss` returns nothing — a stale PID, or sudo -n unavailable — - # every "is it bad?" test misses and the guard reports success having looked - # at nothing. - if ! grep -qE '(^|[[:space:]])127\.0\.0\.1:'"${port}"'([[:space:]]|$)' <<<"$listeners"; then - docker stop "$container" >/dev/null 2>&1 || true - die "Could not confirm ${container} listens on 127.0.0.1:${port}. Container stopped." - fi - if grep -qE ':(80|443)\b' <<<"$listeners"; then - docker stop "$container" >/dev/null 2>&1 || true - die "${container} bound :80/:443 — it would contend with ${CADDY_CONTAINER}. Container stopped." - fi - # Reject *, 0.0.0.0 AND [::] — granian's real default is ::, which a - # 0.0.0.0-only check would sail straight past. - if grep -qE '(\*|0\.0\.0\.0|\[::\]):'"${port}"'\b' <<<"$listeners"; then - docker stop "$container" >/dev/null 2>&1 || true - die "${container} bound ${port} on all interfaces, not loopback. Container stopped." - fi - log_ok "${container} listens on loopback:${port} only — no contention, not LAN-exposed." -} - -start_stack() { - log_info "Starting valkey + searxng…" - docker compose -f "$SEARX_COMPOSE" up -d - # `up -d` is a no-op when only a BIND-MOUNTED config file changed — compose - # compares the compose spec, not the files it mounts. settings.yml and - # limiter.toml are read once at startup, so without an explicit restart the - # container keeps serving the previous config and every verification below - # silently tests the OLD deployment. (Hit for real: limiter.toml was written - # at 15:06 while the container had been running since 15:02, so botdetection - # kept 429ing with "missing config file".) - log_info "Restarting searxng to pick up settings.yml / limiter.toml…" - docker restart "$SEARX_CONTAINER" >/dev/null - - if wait_for_valkey; then - log_ok "valkey answering PING on 127.0.0.1:${VALKEY_PORT}." - else - die "valkey did not become ready — the limiter would silently no-op." - fi - - if wait_for_site; then - log_ok "SearXNG answering on http://127.0.0.1:${SEARX_PORT}/." - else - docker stop "$SEARX_CONTAINER" >/dev/null 2>&1 || true - die "SearXNG did not become reachable on 127.0.0.1:${SEARX_PORT} (container stopped)." - fi - - assert_loopback_only "$VALKEY_CONTAINER" "$VALKEY_PORT" - assert_loopback_only "$SEARX_CONTAINER" "$SEARX_PORT" -} - -# --- Phase 5: front it on the public domain --------------------------------- -# Header syntax matters and is easy to get wrong: -# Header "value" -> "set" (REPLACES any existing value) <-- what we want -# +Header "value" -> "add" (appends, yielding duplicates) -# >Header "value" -> "replace" (substring search/replace WITHIN a value — -# NOT a header setter. Using it here silently -# produced no header at all: it adapts to -# {"replace": {"": [{"search_regexp": "..."}]}} -# and matches nothing. Verified via caddy adapt.) -# This matters because SearXNG sends X-Content-Type-Options and Referrer-Policy -# itself via default_http_headers (its Referrer-Policy is `no-referrer`, which -# differs from ours), so the bare/"set" form is required to override rather than -# duplicate them. -write_snippet() { - cat >"$SEARX_SNIPPET" <' (substring-replace, which sets nothing). - Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' data:; font-src 'self'; connect-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'; manifest-src 'self'" - Strict-Transport-Security "max-age=31536000; includeSubDomains" - X-Content-Type-Options "nosniff" - Referrer-Policy "strict-origin-when-cross-origin" - Permissions-Policy "geolocation=(), microphone=(), camera=(), interest-cohort=()" - # No HSTS preload: duckdns.org is a public suffix, so kuhy.duckdns.org is - # the registrable domain — preloading would force HTTPS on every sibling - # subdomain and is effectively irreversible. Not needed to clear A+. - -Server - } - reverse_proxy 127.0.0.1:${SEARX_PORT} { - # SearXNG's botdetection needs the real client IP. Without this it logs - # "X-Forwarded-For nor X-Real-IP header is set!" and 429s every request, - # including the front page. Paired with trusted_proxies in limiter.toml — - # both halves are required, neither works alone. - header_up X-Forwarded-For {http.request.remote.host} - header_up X-Real-IP {http.request.remote.host} - } -} -EOF - log_ok "Wrote ${SEARX_SNIPPET}." -} - -# --- Phase 6: reload the edge ----------------------------------------------- -reload_caddy() { - if docker exec "$CADDY_CONTAINER" caddy validate --config /etc/caddy/Caddyfile >/dev/null 2>&1; then - docker exec "$CADDY_CONTAINER" caddy reload --config /etc/caddy/Caddyfile - log_ok "Reloaded ${CADDY_CONTAINER}." - else - rm -f "$SEARX_SNIPPET" - die "Caddy config invalid — snippet removed, edge left untouched." - fi -} - -# --- Phase 7: firewall ------------------------------------------------------ -# setup_wireguard_ssh.sh owns /etc/nftables.conf and regenerates it from scratch -# — `flush ruleset` and all — on every run, momentarily dropping the whole -# ruleset. Since all this needs is 80/443 already being accepted, check first -# and only pay for the regeneration when the rule is genuinely missing. -web_ports_open() { - local line ports - while read -r line; do - # Exact token membership: a loose regex would also match "8080, 4433". - ports=",${line//[^0-9,]/}," - if [[ $ports == *,80,* && $ports == *,443,* ]]; then - return 0 - fi - done < <(sudo -n nft list ruleset 2>/dev/null | grep -oE 'tcp dport \{[^}]*\}') - return 1 -} - -ensure_firewall() { - if web_ports_open; then - log_ok "Ports 80/443 already accepted — leaving the ruleset untouched." - return 0 - fi - log_info "Opening ports 80/443 via setup_wireguard_ssh.sh…" - sudo bash "$WG_SCRIPT" allow-web -} - -# --- Phase 8: report -------------------------------------------------------- -print_report() { - cat < http_accept_language (omitting this alone 429s) - # Accept-Encoding -> http_accept_encoding (needs gzip or deflate) - # Accept -> http_accept - # Sec-Fetch-* -> http_sec_fetch (302-redirects when invalid) - # Diagnosing which one bit is easy but non-obvious: set `debug: true` under - # general: in settings.yml, restart, and the log prints - # "NOT OK (searx.botdetection.)" naming the exact check. - # (Observed: dropping only Accept-Language turned a healthy 200 into a 429.) - local url="$1" out code size - out="$(curl -s --compressed -o /dev/null -w '%{http_code} %{size_download}' --max-time 10 \ - -A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-Dest: document' \ - ${2:+-H "Host: $2"} "$url" 2>/dev/null || echo "000 0")" - read -r code size <<<"$out" - [[ $code == "200" && ${size:-0} -gt 0 ]] && echo 0 || echo 1 -} - -check_container() { - local name="$1" port="$2" pid listeners - if ! docker ps --format '{{.Names}}' | grep -qx "$name"; then - status_line 1 "${name} container not running" - return - fi - status_line 0 "${name} container running" - pid="$(docker inspect -f '{{.State.Pid}}' "$name" 2>/dev/null || echo "")" - listeners="$(sudo -n ss -ltnp 2>/dev/null | grep "pid=${pid}," || true)" - if [[ -z $listeners ]]; then - status_line 1 "${name}: could not read listeners (needs sudo -n) — binding UNVERIFIED" - elif grep -qE '(\*|0\.0\.0\.0|\[::\]):'"${port}"'\b' <<<"$listeners"; then - status_line 1 "${name} is bound to ALL INTERFACES on ${port} — LAN-exposed" - elif grep -qE '(^|[[:space:]])127\.0\.0\.1:'"${port}"'([[:space:]]|$)' <<<"$listeners"; then - status_line 0 "${name} listens on loopback:${port} only" - else - status_line 1 "${name} not listening on ${port} as expected" - fi -} - -# Duplicated headers are the classic silent A+ killer, so count them rather than -# merely checking presence. -check_headers() { - local headers out h count - # Same browser-header requirement as check_http — a 429 carries none of these - # headers, so a bare curl would report all five as MISSING on a healthy site. - out="$(curl -sI --compressed --max-time 10 \ - -A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: none' -H 'Sec-Fetch-Dest: document' \ - "https://${SEARX_DOMAIN}/" 2>/dev/null || true)" - if [[ -z $out ]]; then - status_line 1 "could not fetch headers from https://${SEARX_DOMAIN}/" - return - fi - headers=(content-security-policy strict-transport-security - x-content-type-options referrer-policy permissions-policy) - for h in "${headers[@]}"; do - count="$(grep -ci "^${h}:" <<<"$out" || true)" - if [[ $count -eq 1 ]]; then - status_line 0 "header ${h}: exactly 1" - elif [[ $count -eq 0 ]]; then - status_line 1 "header ${h}: MISSING" - else - status_line 1 "header ${h}: DUPLICATED (${count}) — costs the A+ grade" - fi - done -} - -status_cmd() { - print_setup_header "SearXNG status" - - if has_cmd docker; then - status_line 0 "docker present" - else - status_line 1 "docker missing" - fi - - # NOTE: deliberately no ownership/mode assertion on settings.yml — the - # entrypoint chowns the bind mount to uid 977, so a "must be kuhy-owned, - # mode 600" check would false-fail on every run after first start. - if [[ -f $SEARX_SETTINGS ]]; then - status_line 0 "settings.yml present" - else - status_line 1 "settings.yml missing — run setup" - fi - # Its absence is not cosmetic: with no limiter.toml there are no - # trusted_proxies, so botdetection 429s every request including the homepage. - if [[ -f $SEARX_LIMITER ]]; then - status_line 0 "limiter.toml present" - else - status_line 1 "limiter.toml missing — botdetection will 429 everything" - fi - - check_container "$VALKEY_CONTAINER" "$VALKEY_PORT" - check_container "$SEARX_CONTAINER" "$SEARX_PORT" - - # The limiter silently no-ops when valkey is unreachable, so a working search - # does NOT prove the limiter works. Check the backend directly. - local ping - ping="$(docker exec "$VALKEY_CONTAINER" valkey-cli -h 127.0.0.1 -p "$VALKEY_PORT" ping 2>/dev/null || true)" - if [[ $ping == "PONG" ]]; then - status_line 0 "valkey reachable — limiter backend live" - else - status_line 1 "valkey NOT reachable — the limiter is silently disabled" - fi - - status_line "$(check_http "http://127.0.0.1:${SEARX_PORT}/" "$SEARX_DOMAIN")" \ - "local SearXNG (127.0.0.1:${SEARX_PORT}, Host: ${SEARX_DOMAIN})" - - if [[ -f $SEARX_SNIPPET ]]; then - status_line 0 "Caddy snippet present" - else - status_line 1 "Caddy snippet missing" - fi - if getent hosts "$SEARX_DOMAIN" >/dev/null; then - status_line 0 "${SEARX_DOMAIN} resolves" - else - status_line 1 "${SEARX_DOMAIN} does not resolve" - fi - status_line "$(check_http "https://${SEARX_DOMAIN}/")" \ - "external https://${SEARX_DOMAIN}/" - - check_headers - - # A loading front page proves nothing about the actual product. Egress is the - # fragile part on this host (bridge networking has no outbound access at all), - # so run a real query and require real results. Uses the loopback JSON API, - # which pass_ip exempts from botdetection. - local json hits - json="$(curl -s --compressed --max-time 30 \ - "http://127.0.0.1:${SEARX_PORT}/search?q=arch+linux&format=json" 2>/dev/null || true)" - hits="$(grep -o '"url"' <<<"$json" | wc -l)" - if [[ ${hits:-0} -gt 0 ]]; then - status_line 0 "live search returned ${hits} results (engines reachable)" - else - status_line 1 "live search returned NO results — check egress/engines" - fi - # Surfaces engines that are failing rather than merely slow; these are the - # candidates for pruning. - local dead - dead="$(grep -oE '"unresponsive_engines": \[\[[^]]*' <<<"$json" | head -c 300 || true)" - [[ -n $dead ]] && log_info "unresponsive engines: ${dead#*: }" - - # After pruning, general search is served by google cse alone (google/bing - # are inactive: true upstream and only answer to !go / !bi). That is a single - # point of failure worth naming explicitly rather than discovering as "no - # results", so check the engine that is actually carrying the category. - if [[ ${hits:-0} -eq 0 ]]; then - log_warn "general search has no working engine — try: !go / !bi ," - log_warn "or re-enable an engine in the 'engines:' block of ${SEARX_SETTINGS}" - fi - - log_info "Not checked here (needs another machine): the JSON API under bot" - log_info "detection. Run from your phone on cellular, not from this host:" - log_info " curl 'https://${SEARX_DOMAIN}/search?q=test&format=json'" -} main() { local cmd="${1:-setup}"