From 213e16836d995741ac810c564726f685630e91ee Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Thu, 22 Oct 2026 21:36:48 +0200 Subject: [PATCH] redeploy_agent_ready_sites: parallel checks with the checker's own budget --verify-only flaked FAIL on all three sites: CHECK_TIMEOUT=10 killed checks that take 2-22 s. The url checks now run concurrently, each with --budget min(45, time left before a 55 s run deadline - 5 s grace); the outer timeout is only a hang backstop. The redeploy steps run concurrently too (systemctl restart now under a timeout), so a full run fits 60 s worst case. --help prints only the header block. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NBScELfMUiieK9ManFbS3Y --- ...y-redeploy-parallel-checks-2026-10-22.json | 39 +++++++ .../tests/test_redeploy_agent_ready_sites.sh | 58 +++++++++- .../features/redeploy_agent_ready_sites.sh | 109 +++++++++++++----- 3 files changed, 171 insertions(+), 35 deletions(-) create mode 100644 docs/superpowers/evidence/agent-ready-redeploy-parallel-checks-2026-10-22.json diff --git a/docs/superpowers/evidence/agent-ready-redeploy-parallel-checks-2026-10-22.json b/docs/superpowers/evidence/agent-ready-redeploy-parallel-checks-2026-10-22.json new file mode 100644 index 00000000..3aeb5c1b --- /dev/null +++ b/docs/superpowers/evidence/agent-ready-redeploy-parallel-checks-2026-10-22.json @@ -0,0 +1,39 @@ +{ + "intent": "redeploy_agent_ready_sites.sh --verify-only stops flaking FAIL on all three sites: the checks run concurrently, each with the checker's own 45 s network budget, and the whole script still ends inside 60 s.", + "scope": [ + "linux_configuration/features/redeploy_agent_ready_sites.sh", + "linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh", + "Non-goal: changing the AgentReady checker in ~/src/utils; no live container restarts" + ], + "changes": [ + "The three url checks run in parallel (in_parallel helper: per-job output files, exit status collected via wait, verdicts printed in fixed site order)", + "Each check passes --budget to the checker: min(45, RUN_DEADLINE(55) - elapsed - 5 s grace); the outer timeout is budget + 5 s and only a hang backstop. Below 3 s left, every site reports UNDETERMINED (no time left) and nothing starts", + "The three redeploy steps also run in parallel and the systemctl restart is now under the 20 s timeout too, so a full run fits 60 s worst case (20 s redeploy + <=35 s checks)", + "--help prints only the header block instead of every comment in the file" + ], + "verification": [ + { + "command": "bash linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh", + "result": "pass", + "evidence": "RED before the change: 18 passed, 6 failed (concurrency barriers, --budget 45, deadline budget, no-time-left). GREEN after: 25 passed, 0 failed" + }, + { + "command": "bash linux_configuration/features/redeploy_agent_ready_sites.sh --verify-only (3 consecutive live runs)", + "result": "pass", + "evidence": "3/3 runs: PASS kuhy.duckdns.org, dice.kuhy.duckdns.org, popes.kuhy.duckdns.org, rc=0, 0.68-0.89 s wall each" + }, + { + "command": "shellcheck -x + shfmt -d on both files", + "result": "pass", + "evidence": "no findings, no diff" + } + ], + "risks": [ + "Parallel docker recreates of the two static sites contend for the docker daemon; they use separate compose dirs, and the 20 s per-step timeout still applies", + "AGENT_READY_RUN_DEADLINE env override exists for the tests; setting it high in production could exceed 60 s" + ], + "rollback": [ + "git revert this commit", + "Re-run bash linux_configuration/features/redeploy_agent_ready_sites.sh --verify-only and confirm three verdict lines" + ] +} diff --git a/linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh b/linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh index 7ffe2d28..4410007e 100755 --- a/linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh +++ b/linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh @@ -19,23 +19,54 @@ readonly WRAPPER="$WRAP_DIR/redeploy_agent_ready_sites.sh" mkdir -p "$WRAP_DIR" "$TEST_TMPDIR/src/utils/agent_ready" cp "$FEATURES_LIB_DIR/../redeploy_agent_ready_sites.sh" "$WRAPPER" -# A sibling setup script that records its argv and exits $FAKE_RC_. +# Barrier: `barrier.sh ` marks as started and waits +# up to 3 s for all three members of . A sequential caller can never +# have all three running at once, so it logs SERIAL instead: a deterministic +# concurrency check with no timing threshold to flake on. +cat >"$TEST_TMPDIR/barrier.sh" <<'EOF' +#!/usr/bin/env bash +mkdir -p "$TEST_TMPDIR/barrier/$1" +: >"$TEST_TMPDIR/barrier/$1/$2" +for _ in $(seq 30); do + n="$(find "$TEST_TMPDIR/barrier/$1" -type f | wc -l)" + [[ $n -ge 3 ]] && exit 0 + sleep 0.1 +done +printf 'SERIAL %s %s\n' "$1" "$2" >>"$TEST_TMPDIR/calls.log" +EOF +chmod +x "$TEST_TMPDIR/barrier.sh" + +# A sibling setup script that records its argv, joins the redeploy barrier, +# sleeps $FAKE_SLEEP and exits $FAKE_RC_. cat >"$WRAP_DIR/setup_personal_website.sh" <<'EOF' #!/usr/bin/env bash name="$(basename "$0" .sh)" printf '%s %s\n' "$name" "$*" >>"$TEST_TMPDIR/calls.log" +"$TEST_TMPDIR/barrier.sh" redeploy "$name" +sleep "${FAKE_SLEEP:-0}" rc_var="FAKE_RC_${name}" exit "${!rc_var:-0}" EOF cp "$WRAP_DIR/setup_personal_website.sh" "$WRAP_DIR/setup_kcd2_dice_solver.sh" -_t_stub systemctl +cat >"$TEST_TMPDIR/bin/systemctl" <<'EOF' +#!/usr/bin/env bash +printf 'systemctl %s\n' "$*" >>"$TEST_TMPDIR/calls.log" +"$TEST_TMPDIR/barrier.sh" redeploy systemctl +EOF +chmod +x "$TEST_TMPDIR/bin/systemctl" -# Checker stub: PASS for kuhy, FAIL lines for dice, unreachable for popes. +# Checker stub: PASS for kuhy (finishing last), FAIL lines for dice, +# unreachable for popes. Every call joins the check barrier first. cat >"$TEST_TMPDIR/src/utils/agent_ready/check.sh" <<'EOF' #!/usr/bin/env bash printf 'check %s\n' "$*" >>"$TEST_TMPDIR/calls.log" +host="${2#*//}" +"$TEST_TMPDIR/barrier.sh" check "${host%%.*}" case "$2" in -*//kuhy.*) exit "${FAKE_KUHY_RC:-0}" ;; +*//kuhy.*) + sleep 0.5 + exit "${FAKE_KUHY_RC:-0}" + ;; *//dice.*) printf 'PASS AR-READ-03 ok\nFAIL AR-READ-02 soft 404\nFAIL AR-FIND-04 no Link\nFAIL AR-READ-02 again\n' exit 1 @@ -47,6 +78,7 @@ chmod +x "$TEST_TMPDIR/src/utils/agent_ready/check.sh" run_wrapper() { : >"$TEST_TMPDIR/calls.log" + rm -rf "$TEST_TMPDIR/barrier" local rc=0 HOME="$TEST_TMPDIR" bash "$WRAPPER" "$@" >"$TEST_TMPDIR/out" 2>&1 || rc=$? echo "$rc" @@ -61,6 +93,11 @@ _t_has "$out" "PASS https://kuhy.duckdns.org" "kuhy verdict" _t_has "$out" "FAIL https://dice.kuhy.duckdns.org (AR-FIND-04,AR-READ-02)" "dice verdict lists unique failing ids" _t_has "$out" "UNDETERMINED https://popes.kuhy.duckdns.org" "popes verdict on checker exit 3" _t_eq 3 "$(wc -l <"$TEST_TMPDIR/out")" "one line per site" +_t_eq 0 "$(grep -c '^SERIAL check' "$TEST_TMPDIR/calls.log" || true)" "the three checks run concurrently" +_t_eq "kuhy dice popes" "$(grep -oE '//[a-z]+' "$TEST_TMPDIR/out" | tr -d / | paste -sd' ')" \ + "verdicts print in site order even when kuhy finishes last" +_t_eq 3 "$(grep -cE '^check url https://[^ ]+ --budget 45$' "$TEST_TMPDIR/calls.log")" \ + "each check gets the checker's full 45 s network budget" printf '== full run\n' _t_eq 1 "$(FAKE_RC_setup_kcd2_dice_solver=1 run_wrapper)" "full run exits 1" @@ -69,9 +106,20 @@ _t_called "setup_kcd2_dice_solver serve" "dice serve ran" _t_called "systemctl --user restart signal-bot-popes-site.service" "popes unit restarted" _t_has "$(cat "$TEST_TMPDIR/out")" "kcd2-dice FAILED (exit 1)" "a failed serve is reported" _t_eq 3 "$(grep -c '^check url' "$TEST_TMPDIR/calls.log")" "a failed serve does not skip the checks" +_t_eq 0 "$(grep -c '^SERIAL redeploy' "$TEST_TMPDIR/calls.log" || true)" "the three redeploy steps run concurrently" + +printf '== the check budget shrinks to fit the run deadline\n' +_t_eq 1 "$(FAKE_SLEEP=2 AGENT_READY_RUN_DEADLINE=12 run_wrapper)" "slow redeploy still checks (dice fails)" +_t_eq 3 "$(grep -cE '^check url https://[^ ]+ --budget [1-5]$' "$TEST_TMPDIR/calls.log")" \ + "checks get only what is left of the deadline" +_t_eq 1 "$(FAKE_SLEEP=2 AGENT_READY_RUN_DEADLINE=8 run_wrapper)" "no time left fails the run" +_t_eq 0 "$(grep -c '^check url' "$TEST_TMPDIR/calls.log" || true)" "no checker starts once the deadline is spent" +_t_eq 3 "$(grep -c '^UNDETERMINED .*no time left' "$TEST_TMPDIR/out")" "each site reports no time left" printf '== exit codes\n' _t_eq 2 "$(run_wrapper --bogus)" "unknown argument exits 2" -_t_has "$(bash "$WRAPPER" --help)" "--verify-only" "help documents --verify-only" +help_text="$(bash "$WRAPPER" --help)" +_t_has "$help_text" "--verify-only" "help documents --verify-only" +_t_eq 0 "$(grep -c 'in_parallel' <<<"$help_text" || true)" "help stops at the header block" _t_report "redeploy agent-ready sites" diff --git a/linux_configuration/features/redeploy_agent_ready_sites.sh b/linux_configuration/features/redeploy_agent_ready_sites.sh index 4124c3ee..1f96694d 100755 --- a/linux_configuration/features/redeploy_agent_ready_sites.sh +++ b/linux_configuration/features/redeploy_agent_ready_sites.sh @@ -3,16 +3,17 @@ # redeploy_agent_ready_sites.sh — narrow redeploy + AgentReady check of the # three public static sites. # -# Redeploy (skipped with --verify-only), each step independent of the others: +# Redeploy (skipped with --verify-only), the three steps run concurrently: # - setup_personal_website.sh serve recreate the kuhy.duckdns.org container # - setup_kcd2_dice_solver.sh serve recreate the dice.kuhy.duckdns.org one # - systemctl --user restart signal-bot-popes-site.service # None of these uses sudo, pacman, a rebuild or the shared edge: build first in # each site's own repo, then run this to make the new dist/ + Caddyfile live. # -# Verify: ~/src/utils/agent_ready/check.sh url against each public URL, one -# verdict line per site (PASS, FAIL with the failing check ids, or UNDETERMINED -# when the checker could not reach the site). +# Verify: ~/src/utils/agent_ready/check.sh url against each public URL, all +# three at once, one verdict line per site in a fixed order (PASS, FAIL with +# the failing check ids, or UNDETERMINED when the checker could not reach the +# site or ran out of time). # # Usage: # redeploy_agent_ready_sites.sh Redeploy, then verify. @@ -33,60 +34,109 @@ readonly SITES=( "https://dice.kuhy.duckdns.org" "https://popes.kuhy.duckdns.org" ) -# Per-call ceilings keep the whole run inside the 60 s budget even when a site -# hangs; in practice a check takes under a second and a recreate a few. -readonly CHECK_TIMEOUT=10 +# Timing. A live check takes anywhere from under a second to ~22 s (measured), +# so each one gets the checker's own 45 s network budget, which it enforces +# itself and answers with exit 3 when spent. The whole run must end inside +# 60 s, so that budget is cut to whatever is left before RUN_DEADLINE once the +# redeploy is done. GRACE covers Python start-up and the post-network scoring +# and is also the margin of the outer `timeout`, a backstop for a hung process. +readonly RUN_DEADLINE="${AGENT_READY_RUN_DEADLINE:-55}" +readonly CHECK_BUDGET_MAX=45 +readonly CHECK_BUDGET_MIN=3 +readonly GRACE=5 readonly SERVE_TIMEOUT=20 FAILED=0 +CHECK_BUDGET=0 +WORK_DIR="" usage() { - grep -E '^#( |$)' "$0" | sed -E 's/^# ?//' + # The header block only: it ends at the first non-comment line. + sed -nE '2,/^[^#]/{/^#( |$)/s/^# ?//p}' "$0" exit "${1:-0}" } +cleanup() { + [[ -z $WORK_DIR ]] || rm -rf "$WORK_DIR" +} + +# in_parallel ...: run ` ` for every arg at once, then print +# each one's output in argument order. Runs in the background cannot set +# FAILED themselves, so their exit status is collected here. +in_parallel() { + local fn="$1" arg i=0 pids=() + shift + for arg in "$@"; do + "$fn" "$arg" >"${WORK_DIR}/${i}.out" 2>&1 & + pids+=("$!") + i=$((i + 1)) + done + for i in "${!pids[@]}"; do + wait "${pids[$i]}" || FAILED=1 + cat "${WORK_DIR}/${i}.out" + done +} + # run_step