redeploy_agent_ready_sites: parallel checks with the checker's own budget

--verify-only flaked FAIL on all three sites: CHECK_TIMEOUT=10 killed
checks that take 2-22 s. The url checks now run concurrently, each with
--budget min(45, time left before a 55 s run deadline - 5 s grace); the
outer timeout is only a hang backstop. The redeploy steps run
concurrently too (systemctl restart now under a timeout), so a full run
fits 60 s worst case. --help prints only the header block.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NBScELfMUiieK9ManFbS3Y

docs/superpowers/evidence/agent-ready-redeploy-parallel-checks-2026-10-22.json

@@ -0,0 +1,39 @@
+{
+  "intent": "redeploy_agent_ready_sites.sh --verify-only stops flaking FAIL on all three sites: the checks run concurrently, each with the checker's own 45 s network budget, and the whole script still ends inside 60 s.",
+  "scope": [
+    "linux_configuration/features/redeploy_agent_ready_sites.sh",
+    "linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh",
+    "Non-goal: changing the AgentReady checker in ~/src/utils; no live container restarts"
+  ],
+  "changes": [
+    "The three url checks run in parallel (in_parallel helper: per-job output files, exit status collected via wait, verdicts printed in fixed site order)",
+    "Each check passes --budget to the checker: min(45, RUN_DEADLINE(55) - elapsed - 5 s grace); the outer timeout is budget + 5 s and only a hang backstop. Below 3 s left, every site reports UNDETERMINED (no time left) and nothing starts",
+    "The three redeploy steps also run in parallel and the systemctl restart is now under the 20 s timeout too, so a full run fits 60 s worst case (20 s redeploy + <=35 s checks)",
+    "--help prints only the header block instead of every comment in the file"
+  ],
+  "verification": [
+    {
+      "command": "bash linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh",
+      "result": "pass",
+      "evidence": "RED before the change: 18 passed, 6 failed (concurrency barriers, --budget 45, deadline budget, no-time-left). GREEN after: 25 passed, 0 failed"
+    },
+    {
+      "command": "bash linux_configuration/features/redeploy_agent_ready_sites.sh --verify-only (3 consecutive live runs)",
+      "result": "pass",
+      "evidence": "3/3 runs: PASS kuhy.duckdns.org, dice.kuhy.duckdns.org, popes.kuhy.duckdns.org, rc=0, 0.68-0.89 s wall each"
+    },
+    {
+      "command": "shellcheck -x + shfmt -d on both files",
+      "result": "pass",
+      "evidence": "no findings, no diff"
+    }
+  ],
+  "risks": [
+    "Parallel docker recreates of the two static sites contend for the docker daemon; they use separate compose dirs, and the 20 s per-step timeout still applies",
+    "AGENT_READY_RUN_DEADLINE env override exists for the tests; setting it high in production could exceed 60 s"
+  ],
+  "rollback": [
+    "git revert this commit",
+    "Re-run bash linux_configuration/features/redeploy_agent_ready_sites.sh --verify-only and confirm three verdict lines"
+  ]
+}

linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh

@@ -19,23 +19,54 @@ readonly WRAPPER="$WRAP_DIR/redeploy_agent_ready_sites.sh"
 mkdir -p "$WRAP_DIR" "$TEST_TMPDIR/src/utils/agent_ready"
 cp "$FEATURES_LIB_DIR/../redeploy_agent_ready_sites.sh" "$WRAPPER"
 
-# A sibling setup script that records its argv and exits $FAKE_RC_<name>.
+# Barrier: `barrier.sh <group> <member>` marks <member> as started and waits
+# up to 3 s for all three members of <group>. A sequential caller can never
+# have all three running at once, so it logs SERIAL instead: a deterministic
+# concurrency check with no timing threshold to flake on.
+cat >"$TEST_TMPDIR/barrier.sh" <<'EOF'
+#!/usr/bin/env bash
+mkdir -p "$TEST_TMPDIR/barrier/$1"
+: >"$TEST_TMPDIR/barrier/$1/$2"
+for _ in $(seq 30); do
+	n="$(find "$TEST_TMPDIR/barrier/$1" -type f | wc -l)"
+	[[ $n -ge 3 ]] && exit 0
+	sleep 0.1
+done
+printf 'SERIAL %s %s\n' "$1" "$2" >>"$TEST_TMPDIR/calls.log"
+EOF
+chmod +x "$TEST_TMPDIR/barrier.sh"
+
+# A sibling setup script that records its argv, joins the redeploy barrier,
+# sleeps $FAKE_SLEEP and exits $FAKE_RC_<name>.
 cat >"$WRAP_DIR/setup_personal_website.sh" <<'EOF'
 #!/usr/bin/env bash
 name="$(basename "$0" .sh)"
 printf '%s %s\n' "$name" "$*" >>"$TEST_TMPDIR/calls.log"
+"$TEST_TMPDIR/barrier.sh" redeploy "$name"
+sleep "${FAKE_SLEEP:-0}"
 rc_var="FAKE_RC_${name}"
 exit "${!rc_var:-0}"
 EOF
 cp "$WRAP_DIR/setup_personal_website.sh" "$WRAP_DIR/setup_kcd2_dice_solver.sh"
-_t_stub systemctl
+cat >"$TEST_TMPDIR/bin/systemctl" <<'EOF'
+#!/usr/bin/env bash
+printf 'systemctl %s\n' "$*" >>"$TEST_TMPDIR/calls.log"
+"$TEST_TMPDIR/barrier.sh" redeploy systemctl
+EOF
+chmod +x "$TEST_TMPDIR/bin/systemctl"
 
-# Checker stub: PASS for kuhy, FAIL lines for dice, unreachable for popes.
+# Checker stub: PASS for kuhy (finishing last), FAIL lines for dice,
+# unreachable for popes. Every call joins the check barrier first.
 cat >"$TEST_TMPDIR/src/utils/agent_ready/check.sh" <<'EOF'
 #!/usr/bin/env bash
 printf 'check %s\n' "$*" >>"$TEST_TMPDIR/calls.log"
+host="${2#*//}"
+"$TEST_TMPDIR/barrier.sh" check "${host%%.*}"
 case "$2" in
-*//kuhy.*) exit "${FAKE_KUHY_RC:-0}" ;;
+*//kuhy.*)
+	sleep 0.5
+	exit "${FAKE_KUHY_RC:-0}"
+	;;
 *//dice.*)
 	printf 'PASS AR-READ-03 ok\nFAIL AR-READ-02 soft 404\nFAIL AR-FIND-04 no Link\nFAIL AR-READ-02 again\n'
 	exit 1
@@ -47,6 +78,7 @@ chmod +x "$TEST_TMPDIR/src/utils/agent_ready/check.sh"
 
 run_wrapper() {
 	: >"$TEST_TMPDIR/calls.log"
+	rm -rf "$TEST_TMPDIR/barrier"
 	local rc=0
 	HOME="$TEST_TMPDIR" bash "$WRAPPER" "$@" >"$TEST_TMPDIR/out" 2>&1 || rc=$?
 	echo "$rc"
@@ -61,6 +93,11 @@ _t_has "$out" "PASS          https://kuhy.duckdns.org" "kuhy verdict"
 _t_has "$out" "FAIL          https://dice.kuhy.duckdns.org (AR-FIND-04,AR-READ-02)" "dice verdict lists unique failing ids"
 _t_has "$out" "UNDETERMINED  https://popes.kuhy.duckdns.org" "popes verdict on checker exit 3"
 _t_eq 3 "$(wc -l <"$TEST_TMPDIR/out")" "one line per site"
+_t_eq 0 "$(grep -c '^SERIAL check' "$TEST_TMPDIR/calls.log" || true)" "the three checks run concurrently"
+_t_eq "kuhy dice popes" "$(grep -oE '//[a-z]+' "$TEST_TMPDIR/out" | tr -d / | paste -sd' ')" \
+	"verdicts print in site order even when kuhy finishes last"
+_t_eq 3 "$(grep -cE '^check url https://[^ ]+ --budget 45$' "$TEST_TMPDIR/calls.log")" \
+	"each check gets the checker's full 45 s network budget"
 
 printf '== full run\n'
 _t_eq 1 "$(FAKE_RC_setup_kcd2_dice_solver=1 run_wrapper)" "full run exits 1"
@@ -69,9 +106,20 @@ _t_called "setup_kcd2_dice_solver serve" "dice serve ran"
 _t_called "systemctl --user restart signal-bot-popes-site.service" "popes unit restarted"
 _t_has "$(cat "$TEST_TMPDIR/out")" "kcd2-dice                    FAILED (exit 1)" "a failed serve is reported"
 _t_eq 3 "$(grep -c '^check url' "$TEST_TMPDIR/calls.log")" "a failed serve does not skip the checks"
+_t_eq 0 "$(grep -c '^SERIAL redeploy' "$TEST_TMPDIR/calls.log" || true)" "the three redeploy steps run concurrently"
+
+printf '== the check budget shrinks to fit the run deadline\n'
+_t_eq 1 "$(FAKE_SLEEP=2 AGENT_READY_RUN_DEADLINE=12 run_wrapper)" "slow redeploy still checks (dice fails)"
+_t_eq 3 "$(grep -cE '^check url https://[^ ]+ --budget [1-5]$' "$TEST_TMPDIR/calls.log")" \
+	"checks get only what is left of the deadline"
+_t_eq 1 "$(FAKE_SLEEP=2 AGENT_READY_RUN_DEADLINE=8 run_wrapper)" "no time left fails the run"
+_t_eq 0 "$(grep -c '^check url' "$TEST_TMPDIR/calls.log" || true)" "no checker starts once the deadline is spent"
+_t_eq 3 "$(grep -c '^UNDETERMINED .*no time left' "$TEST_TMPDIR/out")" "each site reports no time left"
 
 printf '== exit codes\n'
 _t_eq 2 "$(run_wrapper --bogus)" "unknown argument exits 2"
-_t_has "$(bash "$WRAPPER" --help)" "--verify-only" "help documents --verify-only"
+help_text="$(bash "$WRAPPER" --help)"
+_t_has "$help_text" "--verify-only" "help documents --verify-only"
+_t_eq 0 "$(grep -c 'in_parallel' <<<"$help_text" || true)" "help stops at the header block"
 
 _t_report "redeploy agent-ready sites"

linux_configuration/features/redeploy_agent_ready_sites.sh

@@ -3,16 +3,17 @@
 # redeploy_agent_ready_sites.sh — narrow redeploy + AgentReady check of the
 # three public static sites.
 #
-# Redeploy (skipped with --verify-only), each step independent of the others:
+# Redeploy (skipped with --verify-only), the three steps run concurrently:
 #   - setup_personal_website.sh serve   recreate the kuhy.duckdns.org container
 #   - setup_kcd2_dice_solver.sh serve   recreate the dice.kuhy.duckdns.org one
 #   - systemctl --user restart signal-bot-popes-site.service
 # None of these uses sudo, pacman, a rebuild or the shared edge: build first in
 # each site's own repo, then run this to make the new dist/ + Caddyfile live.
 #
-# Verify: ~/src/utils/agent_ready/check.sh url against each public URL, one
+# Verify: ~/src/utils/agent_ready/check.sh url against each public URL, all
-# verdict line per site (PASS, FAIL with the failing check ids, or UNDETERMINED
+# three at once, one verdict line per site in a fixed order (PASS, FAIL with
-# when the checker could not reach the site).
+# the failing check ids, or UNDETERMINED when the checker could not reach the
+# site or ran out of time).
 #
 # Usage:
 #   redeploy_agent_ready_sites.sh                 Redeploy, then verify.
@@ -33,60 +34,109 @@ readonly SITES=(
 	"https://dice.kuhy.duckdns.org"
 	"https://popes.kuhy.duckdns.org"
 )
-# Per-call ceilings keep the whole run inside the 60 s budget even when a site
+# Timing. A live check takes anywhere from under a second to ~22 s (measured),
-# hangs; in practice a check takes under a second and a recreate a few.
+# so each one gets the checker's own 45 s network budget, which it enforces
-readonly CHECK_TIMEOUT=10
+# itself and answers with exit 3 when spent. The whole run must end inside
+# 60 s, so that budget is cut to whatever is left before RUN_DEADLINE once the
+# redeploy is done. GRACE covers Python start-up and the post-network scoring
+# and is also the margin of the outer `timeout`, a backstop for a hung process.
+readonly RUN_DEADLINE="${AGENT_READY_RUN_DEADLINE:-55}"
+readonly CHECK_BUDGET_MAX=45
+readonly CHECK_BUDGET_MIN=3
+readonly GRACE=5
 readonly SERVE_TIMEOUT=20
 
 FAILED=0
+CHECK_BUDGET=0
+WORK_DIR=""
 
 usage() {
-	grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'
+	# The header block only: it ends at the first non-comment line.
+	sed -nE '2,/^[^#]/{/^#( |$)/s/^# ?//p}' "$0"
 	exit "${1:-0}"
 }
 
+cleanup() {
+	[[ -z $WORK_DIR ]] || rm -rf "$WORK_DIR"
+}
+
+# in_parallel <fn> <arg>...: run `<fn> <arg>` for every arg at once, then print
+# each one's output in argument order. Runs in the background cannot set
+# FAILED themselves, so their exit status is collected here.
+in_parallel() {
+	local fn="$1" arg i=0 pids=()
+	shift
+	for arg in "$@"; do
+		"$fn" "$arg" >"${WORK_DIR}/${i}.out" 2>&1 &
+		pids+=("$!")
+		i=$((i + 1))
+	done
+	for i in "${!pids[@]}"; do
+		wait "${pids[$i]}" || FAILED=1
+		cat "${WORK_DIR}/${i}.out"
+	done
+}
+
 # run_step <label> <cmd...>: output is shown only when the step fails, so a
 # clean run stays at one line per step.
 run_step() {
-	local label="$1" log rc=0
+	local label="$1" out rc=0
 	shift
-	log="$(mktemp)"
+	out="$("$@" 2>&1)" || rc=$?
-	"$@" >"$log" 2>&1 || rc=$?
 	if [[ $rc -eq 0 ]]; then
 		printf 'redeploy %-28s ok\n' "$label"
 	else
 		printf 'redeploy %-28s FAILED (exit %d)\n' "$label" "$rc"
-		tail -n 15 "$log" | sed 's/^/    /'
+		tail -n 15 <<<"$out" | sed 's/^/    /'
-		FAILED=1
 	fi
-	rm -f "$log"
+	return "$rc"
 }
 
-redeploy() {
+redeploy_step() {
-	run_step "personal-website" timeout "$SERVE_TIMEOUT" \
+	case "$1" in
-		bash "${SCRIPT_DIR}/setup_personal_website.sh" serve
+	personal-website)
-	run_step "kcd2-dice" timeout "$SERVE_TIMEOUT" \
+		run_step "$1" timeout "$SERVE_TIMEOUT" bash "${SCRIPT_DIR}/setup_personal_website.sh" serve
-		bash "${SCRIPT_DIR}/setup_kcd2_dice_solver.sh" serve
+		;;
-	run_step "$POPES_UNIT" systemctl --user restart "$POPES_UNIT"
+	kcd2-dice)
+		run_step "$1" timeout "$SERVE_TIMEOUT" bash "${SCRIPT_DIR}/setup_kcd2_dice_solver.sh" serve
+		;;
+	*) run_step "$1" timeout "$SERVE_TIMEOUT" systemctl --user restart "$1" ;;
+	esac
 }
 
-# check_site <url>: one verdict line, sets FAILED on anything but a pass.
+# check_site <url>: one verdict line; non-zero on anything but a pass.
 check_site() {
 	local url="$1" out rc=0 ids
-	out="$(timeout "$CHECK_TIMEOUT" "$CHECKER" url "$url" 2>&1)" || rc=$?
+	out="$(timeout "$((CHECK_BUDGET + GRACE))" "$CHECKER" url "$url" --budget "$CHECK_BUDGET" 2>&1)" || rc=$?
 	case "$rc" in
 	0)
 		printf 'PASS          %s\n' "$url"
 		return 0
 		;;
-	3) printf 'UNDETERMINED  %s (checker could not reach it)\n' "$url" ;;
+	3) printf 'UNDETERMINED  %s (checker could not reach it within %ss)\n' "$url" "$CHECK_BUDGET" ;;
-	124) printf 'FAIL          %s (checker timed out after %ss)\n' "$url" "$CHECK_TIMEOUT" ;;
+	124) printf 'FAIL          %s (checker hung past %ss)\n' "$url" "$((CHECK_BUDGET + GRACE))" ;;
 	*)
 		ids="$(grep -oE '^FAIL AR-[A-Z]+-[0-9]+' <<<"$out" | cut -d' ' -f2 | sort -u | paste -sd, || true)"
 		printf 'FAIL          %s (%s)\n' "$url" "${ids:-exit ${rc}}"
 		;;
 	esac
-	FAILED=1
+	return 1
+}
+
+verify() {
+	local url
+	CHECK_BUDGET=$((RUN_DEADLINE - SECONDS - GRACE))
+	if [[ $CHECK_BUDGET -gt $CHECK_BUDGET_MAX ]]; then
+		CHECK_BUDGET=$CHECK_BUDGET_MAX
+	fi
+	if [[ $CHECK_BUDGET -lt $CHECK_BUDGET_MIN ]]; then
+		for url in "${SITES[@]}"; do
+			printf 'UNDETERMINED  %s (no time left before the %ss deadline)\n' "$url" "$RUN_DEADLINE"
+		done
+		FAILED=1
+		return 0
+	fi
+	in_parallel check_site "${SITES[@]}"
 }
 
 main() {
@@ -104,14 +154,13 @@ main() {
 		printf 'AgentReady checker missing: %s\n' "$CHECKER" >&2
 		exit 1
 	}
+	WORK_DIR="$(mktemp -d)"
+	trap cleanup EXIT
 
 	if [[ $verify_only -eq 0 ]]; then
-		redeploy
+		in_parallel redeploy_step personal-website kcd2-dice "$POPES_UNIT"
 	fi
-	local url
+	verify
-	for url in "${SITES[@]}"; do
-		check_site "$url"
-	done
 	return "$FAILED"
 }