feat(wake): W6 — off-host dead-man beacon + pluggable alarm-sink adapter (#910)
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
This commit was merged in pull request #910.
This commit is contained in:
279
packages/mosaic/framework/tools/wake/test-wake-beacon.sh
Executable file
279
packages/mosaic/framework/tools/wake/test-wake-beacon.sh
Executable file
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-wake-beacon.sh — RED-FIRST invariant harness for W6 (EPIC #892): the
|
||||
# off-host dead-man beacon emitter + pluggable alarm-sink adapter (beacon.sh, A8).
|
||||
#
|
||||
# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if
|
||||
# that invariant regresses:
|
||||
# B1 beacon emitted each cycle -> MONOTONIC (seq strictly increases) (§1.3)
|
||||
# B2 beacon ABSENCE past SLO -> alarm FIRES + ROUTES (stopped emitter);
|
||||
# and NEVER-received -> alarm too — depends on nothing the dying host does (§1.3/G1)
|
||||
# B3 unconfigured OR unreachable ALARM target -> FAIL LOUD (no silent
|
||||
# no-alarm host) (§4 G1/G2a)
|
||||
# B4 isolated host (no off-host monitor) -> DEGRADED different-supervision-root
|
||||
# beacon FLAGGED, never silently "healthy" (§1.3)
|
||||
# B5 same-host-sibling declaration -> REJECTED as non-independent, and the
|
||||
# monotonic seq is NOT advanced by a rejected emit (§1.3)
|
||||
# B6 the alarm/beacon target is resolved BY NAME by the operator adapter;
|
||||
# beacon.sh inlines NO endpoint/secret (§1.4)
|
||||
# B7 unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD (§4 G2a)
|
||||
# B8 no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) (design law)
|
||||
# B9 capture-pane hint is a liveness HINT ONLY -> does NOT suppress absence (§1.3)
|
||||
# B10 fresh beacon within SLO -> ALIVE (exit 0, no alarm) (§1.3)
|
||||
#
|
||||
# Uses per-test isolated XDG homes + pluggable sink/alarm adapter SCRIPTS so a
|
||||
# stopped/dropping emitter and an unreachable target can be exercised. No live
|
||||
# network, no operator queue touched.
|
||||
#
|
||||
# SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell
|
||||
# and re-exports the per-test env, so environments are isolated by design (the
|
||||
# same idiom as test-wake-fn-oracle.sh / test-wake-detector.sh).
|
||||
# shellcheck disable=SC2030,SC2031
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BEACON="$SCRIPT_DIR/beacon.sh"
|
||||
|
||||
command -v jq >/dev/null 2>&1 || {
|
||||
echo "SKIP: jq not available" >&2
|
||||
exit 0
|
||||
}
|
||||
|
||||
TMP_ROOT="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_ROOT"' EXIT
|
||||
|
||||
# Failures recorded to a FILE (subshell-safe — a var counter would silently
|
||||
# swallow failures across the per-test subshells; mirrors the W2/W4/W5 harnesses).
|
||||
FAILFILE="$TMP_ROOT/failures"
|
||||
: >"$FAILFILE"
|
||||
pass=0
|
||||
fail_msg() {
|
||||
echo " FAIL: $*" >&2
|
||||
echo "x" >>"$FAILFILE"
|
||||
}
|
||||
ok() { pass=$((pass + 1)); }
|
||||
|
||||
fresh_home() {
|
||||
local d="$TMP_ROOT/$1"
|
||||
rm -rf "$d"
|
||||
mkdir -p "$d"
|
||||
printf '%s' "$d"
|
||||
}
|
||||
|
||||
# --- reusable sink/alarm adapter scripts (the pluggable operator seam) -------
|
||||
|
||||
# discard-sink: a REACHABLE off-host transport that discards (exit 0).
|
||||
DISCARD_SINK="$TMP_ROOT/discard-sink.sh"
|
||||
cat >"$DISCARD_SINK" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
cat >/dev/null
|
||||
EOF
|
||||
chmod +x "$DISCARD_SINK"
|
||||
|
||||
# recorder-sink: a REACHABLE transport that feeds the received beacon into the
|
||||
# off-host monitor's received store via beacon.sh record (WAKE_BEACON_RECEIVED).
|
||||
RECORDER_SINK="$TMP_ROOT/recorder-sink.sh"
|
||||
cat >"$RECORDER_SINK" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
exec "$BEACON" record
|
||||
EOF
|
||||
chmod +x "$RECORDER_SINK"
|
||||
|
||||
# capture-alarm: a REACHABLE alarm route that writes the routed payload to
|
||||
# \$ALARM_OUT (so a test can prove the alarm actually reached the sink).
|
||||
CAPTURE_ALARM="$TMP_ROOT/capture-alarm.sh"
|
||||
cat >"$CAPTURE_ALARM" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
cat >"$ALARM_OUT"
|
||||
EOF
|
||||
chmod +x "$CAPTURE_ALARM"
|
||||
|
||||
# byname-alarm: resolves its TARGET endpoint BY NAME from a credential store
|
||||
# (the load_credentials shape) — never an inline literal — then "delivers" the
|
||||
# alarm to that resolved target, writing "<target>\n<payload>" to \$ALARM_OUT.
|
||||
BYNAME_ALARM="$TMP_ROOT/byname-alarm.sh"
|
||||
cat >"$BYNAME_ALARM" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -u
|
||||
# The operator adapter owns the endpoint: resolved BY NAME from the cred store,
|
||||
# never passed inline by the framework.
|
||||
target="$(jq -r --arg n "$ALARM_TARGET_NAME" '.wake.beacon_alarm_targets[$n] // empty' "$CRED_FILE")"
|
||||
[ -n "$target" ] || { echo "byname-alarm: target name '$ALARM_TARGET_NAME' not found" >&2; exit 1; }
|
||||
{ printf '%s\n' "$target"; cat; } >"$ALARM_OUT"
|
||||
EOF
|
||||
chmod +x "$BYNAME_ALARM"
|
||||
|
||||
echo "== B1: beacon emitted each cycle -> MONOTONIC (seq strictly increases) =="
|
||||
(
|
||||
WAKE_STATE_HOME="$(fresh_home b1)"
|
||||
export WAKE_STATE_HOME
|
||||
export WAKE_BEACON_INDEPENDENCE="off-host"
|
||||
export WAKE_BEACON_SINK_CMD="$DISCARD_SINK"
|
||||
s1="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')"
|
||||
s2="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')"
|
||||
s3="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')"
|
||||
[ -n "$s1" ] && [ -n "$s2" ] && [ -n "$s3" ] || fail_msg "B1: emit must print a beacon_seq each cycle [$s1/$s2/$s3]"
|
||||
{ [ "$s2" -gt "$s1" ] && [ "$s3" -gt "$s2" ]; } || fail_msg "B1: beacon seq must STRICTLY increase each cycle (got $s1,$s2,$s3)"
|
||||
) && ok
|
||||
|
||||
echo "== B2: beacon ABSENCE past SLO -> alarm FIRES + ROUTES =="
|
||||
(
|
||||
H="$(fresh_home b2)"
|
||||
export ALARM_OUT="$H/alarm.json"
|
||||
export WAKE_BEACON_RECEIVED="$H/received.json"
|
||||
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
|
||||
# Simulate a STOPPED emitter: the monitor's last received beacon is old.
|
||||
now="$(date +%s)"
|
||||
jq -cn --argjson ts "$((now - 100))" \
|
||||
'{kind:"wake-beacon", beacon_seq:5, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \
|
||||
>"$WAKE_BEACON_RECEIVED"
|
||||
out="$("$BEACON" check --slo-seconds 5 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -eq 1 ] || fail_msg "B2: a stale-past-SLO beacon must FIRE the absence alarm (exit 1); got $rc [$out]"
|
||||
echo "$out" | grep -qi 'ALARM FIRED' || fail_msg "B2: absence must announce the alarm fired [$out]"
|
||||
[ -s "$ALARM_OUT" ] || fail_msg "B2: the alarm must actually ROUTE to the sink (payload not written)"
|
||||
jq -e '.kind == "beacon-absence-alarm"' "$ALARM_OUT" >/dev/null 2>&1 || fail_msg "B2: routed payload must be a beacon-absence-alarm [$(cat "$ALARM_OUT" 2>/dev/null)]"
|
||||
# NEVER-received is also an absence (depends on nothing the dying host does).
|
||||
rm -f "$ALARM_OUT"
|
||||
export WAKE_BEACON_RECEIVED="$H/nonexistent.json"
|
||||
out2="$("$BEACON" check --slo-seconds 5 2>&1)"
|
||||
rc2=$?
|
||||
[ "$rc2" -eq 1 ] || fail_msg "B2: a never-received beacon must FIRE the absence alarm (exit 1); got $rc2 [$out2]"
|
||||
[ -s "$ALARM_OUT" ] || fail_msg "B2: never-received absence must route to the alarm sink"
|
||||
) && ok
|
||||
|
||||
echo "== B3: unconfigured OR unreachable ALARM target -> FAIL LOUD =="
|
||||
(
|
||||
H="$(fresh_home b3)"
|
||||
export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence condition
|
||||
# (a) UNCONFIGURED alarm sink.
|
||||
unset WAKE_ALARM_SINK_CMD
|
||||
out="$("$BEACON" check --slo-seconds 5 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -eq 3 ] || fail_msg "B3a: unconfigured alarm target must FAIL LOUD (exit 3); got $rc [$out]"
|
||||
echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B3a: the failure must name the silent-no-alarm-host hazard [$out]"
|
||||
# (b) UNREACHABLE alarm sink (non-zero exit).
|
||||
export WAKE_ALARM_SINK_CMD="false"
|
||||
out2="$("$BEACON" check --slo-seconds 5 2>&1)"
|
||||
rc2=$?
|
||||
[ "$rc2" -eq 3 ] || fail_msg "B3b: unreachable alarm target must FAIL LOUD (exit 3); got $rc2 [$out2]"
|
||||
echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B3b: the failure must name the unreachable target [$out2]"
|
||||
) && ok
|
||||
|
||||
echo "== B4: isolated host -> DEGRADED different-supervision-root beacon FLAGGED =="
|
||||
(
|
||||
WAKE_STATE_HOME="$(fresh_home b4)"
|
||||
export WAKE_STATE_HOME
|
||||
export WAKE_BEACON_INDEPENDENCE="different-supervision-root"
|
||||
export WAKE_BEACON_SINK_CMD="$DISCARD_SINK"
|
||||
out="$("$BEACON" emit 2>"$TMP_ROOT/b4.err")"
|
||||
rc=$?
|
||||
err="$(cat "$TMP_ROOT/b4.err")"
|
||||
[ "$rc" -eq 0 ] || fail_msg "B4: a different-supervision-root emit must still succeed (exit 0); got $rc [$out][$err]"
|
||||
echo "$err" | grep -qi 'DEGRADED' || fail_msg "B4: a different-supervision-root beacon must be FLAGGED degraded on stderr [$err]"
|
||||
echo "$out" | grep -q 'degraded=true' || fail_msg "B4: emit must NOT silently present a degraded beacon as healthy (degraded=true expected) [$out]"
|
||||
) && ok
|
||||
|
||||
echo "== B5: same-host-sibling -> REJECTED as non-independent, seq NOT advanced =="
|
||||
(
|
||||
WAKE_STATE_HOME="$(fresh_home b5)"
|
||||
export WAKE_STATE_HOME
|
||||
export WAKE_BEACON_INDEPENDENCE="same-host-sibling"
|
||||
export WAKE_BEACON_SINK_CMD="$DISCARD_SINK"
|
||||
out="$("$BEACON" emit 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -ne 0 ] || fail_msg "B5: a same-host-sibling beacon must be REJECTED (non-zero exit) [$out]"
|
||||
echo "$out" | grep -qi 'not an independent' || fail_msg "B5: the rejection must explain it is NOT an independent leg [$out]"
|
||||
# A rejected emit must not have minted a seq (rejection precedes counter bump).
|
||||
seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')"
|
||||
[ "${seq_after:-0}" -eq 0 ] || fail_msg "B5: a rejected emit must NOT advance the monotonic seq (got $seq_after)"
|
||||
) && ok
|
||||
|
||||
echo "== B6: alarm target resolved BY NAME; beacon.sh inlines no endpoint/secret =="
|
||||
(
|
||||
H="$(fresh_home b6)"
|
||||
export ALARM_OUT="$H/alarm.out"
|
||||
export CRED_FILE="$H/credentials.json"
|
||||
export ALARM_TARGET_NAME="primary-monitor"
|
||||
SECRET_TARGET="https://monitor.invalid/alarm/DO-NOT-INLINE-abc123"
|
||||
jq -cn --arg t "$SECRET_TARGET" '{wake:{beacon_alarm_targets:{"primary-monitor":$t}}}' >"$CRED_FILE"
|
||||
export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence -> alarm
|
||||
export WAKE_ALARM_SINK_CMD="$BYNAME_ALARM"
|
||||
out="$("$BEACON" check --slo-seconds 5 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -eq 1 ] || fail_msg "B6: absence via a by-name alarm adapter must fire (exit 1); got $rc [$out]"
|
||||
head -n1 "$ALARM_OUT" 2>/dev/null | grep -qF "$SECRET_TARGET" || fail_msg "B6: the adapter must resolve+route to the BY-NAME target [$(cat "$ALARM_OUT" 2>/dev/null)]"
|
||||
# The framework file must NOT inline the endpoint/secret: it only knows a NAME.
|
||||
grep -qF "$SECRET_TARGET" "$BEACON" && fail_msg "B6: beacon.sh must NOT inline the target endpoint/secret"
|
||||
grep -qF "$SECRET_TARGET" "$WAKE_ALARM_SINK_CMD" && fail_msg "B6: even the adapter must resolve by-name, not inline the secret"
|
||||
true
|
||||
) && ok
|
||||
|
||||
echo "== B7: unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD =="
|
||||
(
|
||||
WAKE_STATE_HOME="$(fresh_home b7)"
|
||||
export WAKE_STATE_HOME
|
||||
export WAKE_BEACON_INDEPENDENCE="off-host"
|
||||
# (a) UNCONFIGURED beacon sink.
|
||||
unset WAKE_BEACON_SINK_CMD
|
||||
out="$("$BEACON" emit 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -eq 3 ] || fail_msg "B7a: unconfigured beacon sink must FAIL LOUD (exit 3); got $rc [$out]"
|
||||
echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B7a: the failure must name the silent-no-alarm-host hazard [$out]"
|
||||
# A refused emit (no sink) must not have minted a seq.
|
||||
seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')"
|
||||
[ "${seq_after:-0}" -eq 0 ] || fail_msg "B7a: an unconfigured-sink emit must NOT advance the seq (got $seq_after)"
|
||||
# (b) UNREACHABLE beacon sink (non-zero exit).
|
||||
export WAKE_BEACON_SINK_CMD="false"
|
||||
out2="$("$BEACON" emit 2>&1)"
|
||||
rc2=$?
|
||||
[ "$rc2" -eq 1 ] || fail_msg "B7b: unreachable beacon sink must FAIL LOUD (exit 1); got $rc2 [$out2]"
|
||||
echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B7b: the failure must name the unreachable sink [$out2]"
|
||||
) && ok
|
||||
|
||||
echo "== B8: no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) =="
|
||||
(
|
||||
H="$(fresh_home b8)"
|
||||
export WAKE_BEACON_RECEIVED="$H/nonexistent.json"
|
||||
export WAKE_ALARM_SINK_CMD="$DISCARD_SINK"
|
||||
out="$("$BEACON" check 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -eq 3 ] || fail_msg "B8: check without an SLO must fail loud (exit 3); got $rc [$out]"
|
||||
echo "$out" | grep -qi 'slo' || fail_msg "B8: the usage error must name the missing SLO [$out]"
|
||||
) && ok
|
||||
|
||||
echo "== B9: capture-pane hint is a liveness HINT ONLY (does NOT suppress absence) =="
|
||||
(
|
||||
H="$(fresh_home b9)"
|
||||
export ALARM_OUT="$H/alarm.json"
|
||||
export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # genuine absence
|
||||
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
|
||||
export WAKE_BEACON_PANE_HINT="agent looks idle-at-prompt in capture-pane"
|
||||
out="$("$BEACON" check --slo-seconds 5 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -eq 1 ] || fail_msg "B9: a capture-pane hint must NOT suppress a real absence alarm (expected exit 1); got $rc [$out]"
|
||||
[ -s "$ALARM_OUT" ] || fail_msg "B9: absence must still route despite a pane hint (readiness != pane scrape)"
|
||||
) && ok
|
||||
|
||||
echo "== B10: fresh beacon within SLO -> ALIVE (exit 0, no alarm) =="
|
||||
(
|
||||
H="$(fresh_home b10)"
|
||||
export ALARM_OUT="$H/alarm.json"
|
||||
export WAKE_BEACON_RECEIVED="$H/received.json"
|
||||
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
|
||||
now="$(date +%s)"
|
||||
jq -cn --argjson ts "$now" \
|
||||
'{kind:"wake-beacon", beacon_seq:7, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \
|
||||
>"$WAKE_BEACON_RECEIVED"
|
||||
out="$("$BEACON" check --slo-seconds 60 2>&1)"
|
||||
rc=$?
|
||||
[ "$rc" -eq 0 ] || fail_msg "B10: a fresh beacon within SLO must be ALIVE (exit 0); got $rc [$out]"
|
||||
echo "$out" | grep -qi 'ALIVE' || fail_msg "B10: a fresh beacon must report ALIVE [$out]"
|
||||
[ ! -s "$ALARM_OUT" ] || fail_msg "B10: a fresh beacon must NOT fire an alarm"
|
||||
) && ok
|
||||
|
||||
echo
|
||||
if [ -s "$FAILFILE" ]; then
|
||||
echo "wake beacon harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "wake beacon harness: all invariants passed ($pass groups)"
|
||||
Reference in New Issue
Block a user