mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 18:07:06 +01:00
redeploy_agent_ready_sites: parallel checks with the checker's own budget
--verify-only flaked FAIL on all three sites: CHECK_TIMEOUT=10 killed checks that take 2-22 s. The url checks now run concurrently, each with --budget min(45, time left before a 55 s run deadline - 5 s grace); the outer timeout is only a hang backstop. The redeploy steps run concurrently too (systemctl restart now under a timeout), so a full run fits 60 s worst case. --help prints only the header block. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NBScELfMUiieK9ManFbS3Y
This commit is contained in:
parent
2a76d05f9d
commit
213e16836d
@ -0,0 +1,39 @@
|
||||
{
|
||||
"intent": "redeploy_agent_ready_sites.sh --verify-only stops flaking FAIL on all three sites: the checks run concurrently, each with the checker's own 45 s network budget, and the whole script still ends inside 60 s.",
|
||||
"scope": [
|
||||
"linux_configuration/features/redeploy_agent_ready_sites.sh",
|
||||
"linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh",
|
||||
"Non-goal: changing the AgentReady checker in ~/src/utils; no live container restarts"
|
||||
],
|
||||
"changes": [
|
||||
"The three url checks run in parallel (in_parallel helper: per-job output files, exit status collected via wait, verdicts printed in fixed site order)",
|
||||
"Each check passes --budget to the checker: min(45, RUN_DEADLINE(55) - elapsed - 5 s grace); the outer timeout is budget + 5 s and only a hang backstop. Below 3 s left, every site reports UNDETERMINED (no time left) and nothing starts",
|
||||
"The three redeploy steps also run in parallel and the systemctl restart is now under the 20 s timeout too, so a full run fits 60 s worst case (20 s redeploy + <=35 s checks)",
|
||||
"--help prints only the header block instead of every comment in the file"
|
||||
],
|
||||
"verification": [
|
||||
{
|
||||
"command": "bash linux_configuration/features/lib/tests/test_redeploy_agent_ready_sites.sh",
|
||||
"result": "pass",
|
||||
"evidence": "RED before the change: 18 passed, 6 failed (concurrency barriers, --budget 45, deadline budget, no-time-left). GREEN after: 25 passed, 0 failed"
|
||||
},
|
||||
{
|
||||
"command": "bash linux_configuration/features/redeploy_agent_ready_sites.sh --verify-only (3 consecutive live runs)",
|
||||
"result": "pass",
|
||||
"evidence": "3/3 runs: PASS kuhy.duckdns.org, dice.kuhy.duckdns.org, popes.kuhy.duckdns.org, rc=0, 0.68-0.89 s wall each"
|
||||
},
|
||||
{
|
||||
"command": "shellcheck -x + shfmt -d on both files",
|
||||
"result": "pass",
|
||||
"evidence": "no findings, no diff"
|
||||
}
|
||||
],
|
||||
"risks": [
|
||||
"Parallel docker recreates of the two static sites contend for the docker daemon; they use separate compose dirs, and the 20 s per-step timeout still applies",
|
||||
"AGENT_READY_RUN_DEADLINE env override exists for the tests; setting it high in production could exceed 60 s"
|
||||
],
|
||||
"rollback": [
|
||||
"git revert this commit",
|
||||
"Re-run bash linux_configuration/features/redeploy_agent_ready_sites.sh --verify-only and confirm three verdict lines"
|
||||
]
|
||||
}
|
||||
@ -19,23 +19,54 @@ readonly WRAPPER="$WRAP_DIR/redeploy_agent_ready_sites.sh"
|
||||
mkdir -p "$WRAP_DIR" "$TEST_TMPDIR/src/utils/agent_ready"
|
||||
cp "$FEATURES_LIB_DIR/../redeploy_agent_ready_sites.sh" "$WRAPPER"
|
||||
|
||||
# A sibling setup script that records its argv and exits $FAKE_RC_<name>.
|
||||
# Barrier: `barrier.sh <group> <member>` marks <member> as started and waits
|
||||
# up to 3 s for all three members of <group>. A sequential caller can never
|
||||
# have all three running at once, so it logs SERIAL instead: a deterministic
|
||||
# concurrency check with no timing threshold to flake on.
|
||||
cat >"$TEST_TMPDIR/barrier.sh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
mkdir -p "$TEST_TMPDIR/barrier/$1"
|
||||
: >"$TEST_TMPDIR/barrier/$1/$2"
|
||||
for _ in $(seq 30); do
|
||||
n="$(find "$TEST_TMPDIR/barrier/$1" -type f | wc -l)"
|
||||
[[ $n -ge 3 ]] && exit 0
|
||||
sleep 0.1
|
||||
done
|
||||
printf 'SERIAL %s %s\n' "$1" "$2" >>"$TEST_TMPDIR/calls.log"
|
||||
EOF
|
||||
chmod +x "$TEST_TMPDIR/barrier.sh"
|
||||
|
||||
# A sibling setup script that records its argv, joins the redeploy barrier,
|
||||
# sleeps $FAKE_SLEEP and exits $FAKE_RC_<name>.
|
||||
cat >"$WRAP_DIR/setup_personal_website.sh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
name="$(basename "$0" .sh)"
|
||||
printf '%s %s\n' "$name" "$*" >>"$TEST_TMPDIR/calls.log"
|
||||
"$TEST_TMPDIR/barrier.sh" redeploy "$name"
|
||||
sleep "${FAKE_SLEEP:-0}"
|
||||
rc_var="FAKE_RC_${name}"
|
||||
exit "${!rc_var:-0}"
|
||||
EOF
|
||||
cp "$WRAP_DIR/setup_personal_website.sh" "$WRAP_DIR/setup_kcd2_dice_solver.sh"
|
||||
_t_stub systemctl
|
||||
cat >"$TEST_TMPDIR/bin/systemctl" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf 'systemctl %s\n' "$*" >>"$TEST_TMPDIR/calls.log"
|
||||
"$TEST_TMPDIR/barrier.sh" redeploy systemctl
|
||||
EOF
|
||||
chmod +x "$TEST_TMPDIR/bin/systemctl"
|
||||
|
||||
# Checker stub: PASS for kuhy, FAIL lines for dice, unreachable for popes.
|
||||
# Checker stub: PASS for kuhy (finishing last), FAIL lines for dice,
|
||||
# unreachable for popes. Every call joins the check barrier first.
|
||||
cat >"$TEST_TMPDIR/src/utils/agent_ready/check.sh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf 'check %s\n' "$*" >>"$TEST_TMPDIR/calls.log"
|
||||
host="${2#*//}"
|
||||
"$TEST_TMPDIR/barrier.sh" check "${host%%.*}"
|
||||
case "$2" in
|
||||
*//kuhy.*) exit "${FAKE_KUHY_RC:-0}" ;;
|
||||
*//kuhy.*)
|
||||
sleep 0.5
|
||||
exit "${FAKE_KUHY_RC:-0}"
|
||||
;;
|
||||
*//dice.*)
|
||||
printf 'PASS AR-READ-03 ok\nFAIL AR-READ-02 soft 404\nFAIL AR-FIND-04 no Link\nFAIL AR-READ-02 again\n'
|
||||
exit 1
|
||||
@ -47,6 +78,7 @@ chmod +x "$TEST_TMPDIR/src/utils/agent_ready/check.sh"
|
||||
|
||||
run_wrapper() {
|
||||
: >"$TEST_TMPDIR/calls.log"
|
||||
rm -rf "$TEST_TMPDIR/barrier"
|
||||
local rc=0
|
||||
HOME="$TEST_TMPDIR" bash "$WRAPPER" "$@" >"$TEST_TMPDIR/out" 2>&1 || rc=$?
|
||||
echo "$rc"
|
||||
@ -61,6 +93,11 @@ _t_has "$out" "PASS https://kuhy.duckdns.org" "kuhy verdict"
|
||||
_t_has "$out" "FAIL https://dice.kuhy.duckdns.org (AR-FIND-04,AR-READ-02)" "dice verdict lists unique failing ids"
|
||||
_t_has "$out" "UNDETERMINED https://popes.kuhy.duckdns.org" "popes verdict on checker exit 3"
|
||||
_t_eq 3 "$(wc -l <"$TEST_TMPDIR/out")" "one line per site"
|
||||
_t_eq 0 "$(grep -c '^SERIAL check' "$TEST_TMPDIR/calls.log" || true)" "the three checks run concurrently"
|
||||
_t_eq "kuhy dice popes" "$(grep -oE '//[a-z]+' "$TEST_TMPDIR/out" | tr -d / | paste -sd' ')" \
|
||||
"verdicts print in site order even when kuhy finishes last"
|
||||
_t_eq 3 "$(grep -cE '^check url https://[^ ]+ --budget 45$' "$TEST_TMPDIR/calls.log")" \
|
||||
"each check gets the checker's full 45 s network budget"
|
||||
|
||||
printf '== full run\n'
|
||||
_t_eq 1 "$(FAKE_RC_setup_kcd2_dice_solver=1 run_wrapper)" "full run exits 1"
|
||||
@ -69,9 +106,20 @@ _t_called "setup_kcd2_dice_solver serve" "dice serve ran"
|
||||
_t_called "systemctl --user restart signal-bot-popes-site.service" "popes unit restarted"
|
||||
_t_has "$(cat "$TEST_TMPDIR/out")" "kcd2-dice FAILED (exit 1)" "a failed serve is reported"
|
||||
_t_eq 3 "$(grep -c '^check url' "$TEST_TMPDIR/calls.log")" "a failed serve does not skip the checks"
|
||||
_t_eq 0 "$(grep -c '^SERIAL redeploy' "$TEST_TMPDIR/calls.log" || true)" "the three redeploy steps run concurrently"
|
||||
|
||||
printf '== the check budget shrinks to fit the run deadline\n'
|
||||
_t_eq 1 "$(FAKE_SLEEP=2 AGENT_READY_RUN_DEADLINE=12 run_wrapper)" "slow redeploy still checks (dice fails)"
|
||||
_t_eq 3 "$(grep -cE '^check url https://[^ ]+ --budget [1-5]$' "$TEST_TMPDIR/calls.log")" \
|
||||
"checks get only what is left of the deadline"
|
||||
_t_eq 1 "$(FAKE_SLEEP=2 AGENT_READY_RUN_DEADLINE=8 run_wrapper)" "no time left fails the run"
|
||||
_t_eq 0 "$(grep -c '^check url' "$TEST_TMPDIR/calls.log" || true)" "no checker starts once the deadline is spent"
|
||||
_t_eq 3 "$(grep -c '^UNDETERMINED .*no time left' "$TEST_TMPDIR/out")" "each site reports no time left"
|
||||
|
||||
printf '== exit codes\n'
|
||||
_t_eq 2 "$(run_wrapper --bogus)" "unknown argument exits 2"
|
||||
_t_has "$(bash "$WRAPPER" --help)" "--verify-only" "help documents --verify-only"
|
||||
help_text="$(bash "$WRAPPER" --help)"
|
||||
_t_has "$help_text" "--verify-only" "help documents --verify-only"
|
||||
_t_eq 0 "$(grep -c 'in_parallel' <<<"$help_text" || true)" "help stops at the header block"
|
||||
|
||||
_t_report "redeploy agent-ready sites"
|
||||
|
||||
@ -3,16 +3,17 @@
|
||||
# redeploy_agent_ready_sites.sh — narrow redeploy + AgentReady check of the
|
||||
# three public static sites.
|
||||
#
|
||||
# Redeploy (skipped with --verify-only), each step independent of the others:
|
||||
# Redeploy (skipped with --verify-only), the three steps run concurrently:
|
||||
# - setup_personal_website.sh serve recreate the kuhy.duckdns.org container
|
||||
# - setup_kcd2_dice_solver.sh serve recreate the dice.kuhy.duckdns.org one
|
||||
# - systemctl --user restart signal-bot-popes-site.service
|
||||
# None of these uses sudo, pacman, a rebuild or the shared edge: build first in
|
||||
# each site's own repo, then run this to make the new dist/ + Caddyfile live.
|
||||
#
|
||||
# Verify: ~/src/utils/agent_ready/check.sh url against each public URL, one
|
||||
# verdict line per site (PASS, FAIL with the failing check ids, or UNDETERMINED
|
||||
# when the checker could not reach the site).
|
||||
# Verify: ~/src/utils/agent_ready/check.sh url against each public URL, all
|
||||
# three at once, one verdict line per site in a fixed order (PASS, FAIL with
|
||||
# the failing check ids, or UNDETERMINED when the checker could not reach the
|
||||
# site or ran out of time).
|
||||
#
|
||||
# Usage:
|
||||
# redeploy_agent_ready_sites.sh Redeploy, then verify.
|
||||
@ -33,60 +34,109 @@ readonly SITES=(
|
||||
"https://dice.kuhy.duckdns.org"
|
||||
"https://popes.kuhy.duckdns.org"
|
||||
)
|
||||
# Per-call ceilings keep the whole run inside the 60 s budget even when a site
|
||||
# hangs; in practice a check takes under a second and a recreate a few.
|
||||
readonly CHECK_TIMEOUT=10
|
||||
# Timing. A live check takes anywhere from under a second to ~22 s (measured),
|
||||
# so each one gets the checker's own 45 s network budget, which it enforces
|
||||
# itself and answers with exit 3 when spent. The whole run must end inside
|
||||
# 60 s, so that budget is cut to whatever is left before RUN_DEADLINE once the
|
||||
# redeploy is done. GRACE covers Python start-up and the post-network scoring
|
||||
# and is also the margin of the outer `timeout`, a backstop for a hung process.
|
||||
readonly RUN_DEADLINE="${AGENT_READY_RUN_DEADLINE:-55}"
|
||||
readonly CHECK_BUDGET_MAX=45
|
||||
readonly CHECK_BUDGET_MIN=3
|
||||
readonly GRACE=5
|
||||
readonly SERVE_TIMEOUT=20
|
||||
|
||||
FAILED=0
|
||||
CHECK_BUDGET=0
|
||||
WORK_DIR=""
|
||||
|
||||
usage() {
|
||||
grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'
|
||||
# The header block only: it ends at the first non-comment line.
|
||||
sed -nE '2,/^[^#]/{/^#( |$)/s/^# ?//p}' "$0"
|
||||
exit "${1:-0}"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
[[ -z $WORK_DIR ]] || rm -rf "$WORK_DIR"
|
||||
}
|
||||
|
||||
# in_parallel <fn> <arg>...: run `<fn> <arg>` for every arg at once, then print
|
||||
# each one's output in argument order. Runs in the background cannot set
|
||||
# FAILED themselves, so their exit status is collected here.
|
||||
in_parallel() {
|
||||
local fn="$1" arg i=0 pids=()
|
||||
shift
|
||||
for arg in "$@"; do
|
||||
"$fn" "$arg" >"${WORK_DIR}/${i}.out" 2>&1 &
|
||||
pids+=("$!")
|
||||
i=$((i + 1))
|
||||
done
|
||||
for i in "${!pids[@]}"; do
|
||||
wait "${pids[$i]}" || FAILED=1
|
||||
cat "${WORK_DIR}/${i}.out"
|
||||
done
|
||||
}
|
||||
|
||||
# run_step <label> <cmd...>: output is shown only when the step fails, so a
|
||||
# clean run stays at one line per step.
|
||||
run_step() {
|
||||
local label="$1" log rc=0
|
||||
local label="$1" out rc=0
|
||||
shift
|
||||
log="$(mktemp)"
|
||||
"$@" >"$log" 2>&1 || rc=$?
|
||||
out="$("$@" 2>&1)" || rc=$?
|
||||
if [[ $rc -eq 0 ]]; then
|
||||
printf 'redeploy %-28s ok\n' "$label"
|
||||
else
|
||||
printf 'redeploy %-28s FAILED (exit %d)\n' "$label" "$rc"
|
||||
tail -n 15 "$log" | sed 's/^/ /'
|
||||
FAILED=1
|
||||
tail -n 15 <<<"$out" | sed 's/^/ /'
|
||||
fi
|
||||
rm -f "$log"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
redeploy() {
|
||||
run_step "personal-website" timeout "$SERVE_TIMEOUT" \
|
||||
bash "${SCRIPT_DIR}/setup_personal_website.sh" serve
|
||||
run_step "kcd2-dice" timeout "$SERVE_TIMEOUT" \
|
||||
bash "${SCRIPT_DIR}/setup_kcd2_dice_solver.sh" serve
|
||||
run_step "$POPES_UNIT" systemctl --user restart "$POPES_UNIT"
|
||||
redeploy_step() {
|
||||
case "$1" in
|
||||
personal-website)
|
||||
run_step "$1" timeout "$SERVE_TIMEOUT" bash "${SCRIPT_DIR}/setup_personal_website.sh" serve
|
||||
;;
|
||||
kcd2-dice)
|
||||
run_step "$1" timeout "$SERVE_TIMEOUT" bash "${SCRIPT_DIR}/setup_kcd2_dice_solver.sh" serve
|
||||
;;
|
||||
*) run_step "$1" timeout "$SERVE_TIMEOUT" systemctl --user restart "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# check_site <url>: one verdict line, sets FAILED on anything but a pass.
|
||||
# check_site <url>: one verdict line; non-zero on anything but a pass.
|
||||
check_site() {
|
||||
local url="$1" out rc=0 ids
|
||||
out="$(timeout "$CHECK_TIMEOUT" "$CHECKER" url "$url" 2>&1)" || rc=$?
|
||||
out="$(timeout "$((CHECK_BUDGET + GRACE))" "$CHECKER" url "$url" --budget "$CHECK_BUDGET" 2>&1)" || rc=$?
|
||||
case "$rc" in
|
||||
0)
|
||||
printf 'PASS %s\n' "$url"
|
||||
return 0
|
||||
;;
|
||||
3) printf 'UNDETERMINED %s (checker could not reach it)\n' "$url" ;;
|
||||
124) printf 'FAIL %s (checker timed out after %ss)\n' "$url" "$CHECK_TIMEOUT" ;;
|
||||
3) printf 'UNDETERMINED %s (checker could not reach it within %ss)\n' "$url" "$CHECK_BUDGET" ;;
|
||||
124) printf 'FAIL %s (checker hung past %ss)\n' "$url" "$((CHECK_BUDGET + GRACE))" ;;
|
||||
*)
|
||||
ids="$(grep -oE '^FAIL AR-[A-Z]+-[0-9]+' <<<"$out" | cut -d' ' -f2 | sort -u | paste -sd, || true)"
|
||||
printf 'FAIL %s (%s)\n' "$url" "${ids:-exit ${rc}}"
|
||||
;;
|
||||
esac
|
||||
FAILED=1
|
||||
return 1
|
||||
}
|
||||
|
||||
verify() {
|
||||
local url
|
||||
CHECK_BUDGET=$((RUN_DEADLINE - SECONDS - GRACE))
|
||||
if [[ $CHECK_BUDGET -gt $CHECK_BUDGET_MAX ]]; then
|
||||
CHECK_BUDGET=$CHECK_BUDGET_MAX
|
||||
fi
|
||||
if [[ $CHECK_BUDGET -lt $CHECK_BUDGET_MIN ]]; then
|
||||
for url in "${SITES[@]}"; do
|
||||
printf 'UNDETERMINED %s (no time left before the %ss deadline)\n' "$url" "$RUN_DEADLINE"
|
||||
done
|
||||
FAILED=1
|
||||
return 0
|
||||
fi
|
||||
in_parallel check_site "${SITES[@]}"
|
||||
}
|
||||
|
||||
main() {
|
||||
@ -104,14 +154,13 @@ main() {
|
||||
printf 'AgentReady checker missing: %s\n' "$CHECKER" >&2
|
||||
exit 1
|
||||
}
|
||||
WORK_DIR="$(mktemp -d)"
|
||||
trap cleanup EXIT
|
||||
|
||||
if [[ $verify_only -eq 0 ]]; then
|
||||
redeploy
|
||||
in_parallel redeploy_step personal-website kcd2-dice "$POPES_UNIT"
|
||||
fi
|
||||
local url
|
||||
for url in "${SITES[@]}"; do
|
||||
check_site "$url"
|
||||
done
|
||||
verify
|
||||
return "$FAILED"
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user