#!/usr/bin/env bash # test-wake-beacon.sh — RED-FIRST invariant harness for W6 (EPIC #892): the # off-host dead-man beacon emitter + pluggable alarm-sink adapter (beacon.sh, A8). # # Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if # that invariant regresses: # B1 beacon emitted each cycle -> MONOTONIC (seq strictly increases) (§1.3) # B2 beacon ABSENCE past SLO -> alarm FIRES + ROUTES (stopped emitter); # and NEVER-received -> alarm too — depends on nothing the dying host does (§1.3/G1) # B3 unconfigured OR unreachable ALARM target -> FAIL LOUD (no silent # no-alarm host) (§4 G1/G2a) # B4 isolated host (no off-host monitor) -> DEGRADED different-supervision-root # beacon FLAGGED, never silently "healthy" (§1.3) # B5 same-host-sibling declaration -> REJECTED as non-independent, and the # monotonic seq is NOT advanced by a rejected emit (§1.3) # B6 the alarm/beacon target is resolved BY NAME by the operator adapter; # beacon.sh inlines NO endpoint/secret (§1.4) # B7 unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD (§4 G2a) # B8 no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) (design law) # B9 capture-pane hint is a liveness HINT ONLY -> does NOT suppress absence (§1.3) # B10 fresh beacon within SLO -> ALIVE (exit 0, no alarm) (§1.3) # # Uses per-test isolated XDG homes + pluggable sink/alarm adapter SCRIPTS so a # stopped/dropping emitter and an unreachable target can be exercised. No live # network, no operator queue touched. # # SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell # and re-exports the per-test env, so environments are isolated by design (the # same idiom as test-wake-fn-oracle.sh / test-wake-detector.sh). # shellcheck disable=SC2030,SC2031 set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BEACON="$SCRIPT_DIR/beacon.sh" command -v jq >/dev/null 2>&1 || { echo "SKIP: jq not available" >&2 exit 0 } TMP_ROOT="$(mktemp -d)" trap 'rm -rf "$TMP_ROOT"' EXIT # Failures recorded to a FILE (subshell-safe — a var counter would silently # swallow failures across the per-test subshells; mirrors the W2/W4/W5 harnesses). FAILFILE="$TMP_ROOT/failures" : >"$FAILFILE" pass=0 fail_msg() { echo " FAIL: $*" >&2 echo "x" >>"$FAILFILE" } ok() { pass=$((pass + 1)); } fresh_home() { local d="$TMP_ROOT/$1" rm -rf "$d" mkdir -p "$d" printf '%s' "$d" } # --- reusable sink/alarm adapter scripts (the pluggable operator seam) ------- # discard-sink: a REACHABLE off-host transport that discards (exit 0). DISCARD_SINK="$TMP_ROOT/discard-sink.sh" cat >"$DISCARD_SINK" <<'EOF' #!/usr/bin/env bash cat >/dev/null EOF chmod +x "$DISCARD_SINK" # recorder-sink: a REACHABLE transport that feeds the received beacon into the # off-host monitor's received store via beacon.sh record (WAKE_BEACON_RECEIVED). RECORDER_SINK="$TMP_ROOT/recorder-sink.sh" cat >"$RECORDER_SINK" <"$CAPTURE_ALARM" <<'EOF' #!/usr/bin/env bash cat >"$ALARM_OUT" EOF chmod +x "$CAPTURE_ALARM" # byname-alarm: resolves its TARGET endpoint BY NAME from a credential store # (the load_credentials shape) — never an inline literal — then "delivers" the # alarm to that resolved target, writing "\n" to \$ALARM_OUT. BYNAME_ALARM="$TMP_ROOT/byname-alarm.sh" cat >"$BYNAME_ALARM" <<'EOF' #!/usr/bin/env bash set -u # The operator adapter owns the endpoint: resolved BY NAME from the cred store, # never passed inline by the framework. target="$(jq -r --arg n "$ALARM_TARGET_NAME" '.wake.beacon_alarm_targets[$n] // empty' "$CRED_FILE")" [ -n "$target" ] || { echo "byname-alarm: target name '$ALARM_TARGET_NAME' not found" >&2; exit 1; } { printf '%s\n' "$target"; cat; } >"$ALARM_OUT" EOF chmod +x "$BYNAME_ALARM" echo "== B1: beacon emitted each cycle -> MONOTONIC (seq strictly increases) ==" ( WAKE_STATE_HOME="$(fresh_home b1)" export WAKE_STATE_HOME export WAKE_BEACON_INDEPENDENCE="off-host" export WAKE_BEACON_SINK_CMD="$DISCARD_SINK" s1="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')" s2="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')" s3="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')" [ -n "$s1" ] && [ -n "$s2" ] && [ -n "$s3" ] || fail_msg "B1: emit must print a beacon_seq each cycle [$s1/$s2/$s3]" { [ "$s2" -gt "$s1" ] && [ "$s3" -gt "$s2" ]; } || fail_msg "B1: beacon seq must STRICTLY increase each cycle (got $s1,$s2,$s3)" ) && ok echo "== B2: beacon ABSENCE past SLO -> alarm FIRES + ROUTES ==" ( H="$(fresh_home b2)" export ALARM_OUT="$H/alarm.json" export WAKE_BEACON_RECEIVED="$H/received.json" export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" # Simulate a STOPPED emitter: the monitor's last received beacon is old. now="$(date +%s)" jq -cn --argjson ts "$((now - 100))" \ '{kind:"wake-beacon", beacon_seq:5, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \ >"$WAKE_BEACON_RECEIVED" out="$("$BEACON" check --slo-seconds 5 2>&1)" rc=$? [ "$rc" -eq 1 ] || fail_msg "B2: a stale-past-SLO beacon must FIRE the absence alarm (exit 1); got $rc [$out]" echo "$out" | grep -qi 'ALARM FIRED' || fail_msg "B2: absence must announce the alarm fired [$out]" [ -s "$ALARM_OUT" ] || fail_msg "B2: the alarm must actually ROUTE to the sink (payload not written)" jq -e '.kind == "beacon-absence-alarm"' "$ALARM_OUT" >/dev/null 2>&1 || fail_msg "B2: routed payload must be a beacon-absence-alarm [$(cat "$ALARM_OUT" 2>/dev/null)]" # NEVER-received is also an absence (depends on nothing the dying host does). rm -f "$ALARM_OUT" export WAKE_BEACON_RECEIVED="$H/nonexistent.json" out2="$("$BEACON" check --slo-seconds 5 2>&1)" rc2=$? [ "$rc2" -eq 1 ] || fail_msg "B2: a never-received beacon must FIRE the absence alarm (exit 1); got $rc2 [$out2]" [ -s "$ALARM_OUT" ] || fail_msg "B2: never-received absence must route to the alarm sink" ) && ok echo "== B3: unconfigured OR unreachable ALARM target -> FAIL LOUD ==" ( H="$(fresh_home b3)" export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence condition # (a) UNCONFIGURED alarm sink. unset WAKE_ALARM_SINK_CMD out="$("$BEACON" check --slo-seconds 5 2>&1)" rc=$? [ "$rc" -eq 3 ] || fail_msg "B3a: unconfigured alarm target must FAIL LOUD (exit 3); got $rc [$out]" echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B3a: the failure must name the silent-no-alarm-host hazard [$out]" # (b) UNREACHABLE alarm sink (non-zero exit). export WAKE_ALARM_SINK_CMD="false" out2="$("$BEACON" check --slo-seconds 5 2>&1)" rc2=$? [ "$rc2" -eq 3 ] || fail_msg "B3b: unreachable alarm target must FAIL LOUD (exit 3); got $rc2 [$out2]" echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B3b: the failure must name the unreachable target [$out2]" ) && ok echo "== B4: isolated host -> DEGRADED different-supervision-root beacon FLAGGED ==" ( WAKE_STATE_HOME="$(fresh_home b4)" export WAKE_STATE_HOME export WAKE_BEACON_INDEPENDENCE="different-supervision-root" export WAKE_BEACON_SINK_CMD="$DISCARD_SINK" out="$("$BEACON" emit 2>"$TMP_ROOT/b4.err")" rc=$? err="$(cat "$TMP_ROOT/b4.err")" [ "$rc" -eq 0 ] || fail_msg "B4: a different-supervision-root emit must still succeed (exit 0); got $rc [$out][$err]" echo "$err" | grep -qi 'DEGRADED' || fail_msg "B4: a different-supervision-root beacon must be FLAGGED degraded on stderr [$err]" echo "$out" | grep -q 'degraded=true' || fail_msg "B4: emit must NOT silently present a degraded beacon as healthy (degraded=true expected) [$out]" ) && ok echo "== B5: same-host-sibling -> REJECTED as non-independent, seq NOT advanced ==" ( WAKE_STATE_HOME="$(fresh_home b5)" export WAKE_STATE_HOME export WAKE_BEACON_INDEPENDENCE="same-host-sibling" export WAKE_BEACON_SINK_CMD="$DISCARD_SINK" out="$("$BEACON" emit 2>&1)" rc=$? [ "$rc" -ne 0 ] || fail_msg "B5: a same-host-sibling beacon must be REJECTED (non-zero exit) [$out]" echo "$out" | grep -qi 'not an independent' || fail_msg "B5: the rejection must explain it is NOT an independent leg [$out]" # A rejected emit must not have minted a seq (rejection precedes counter bump). seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')" [ "${seq_after:-0}" -eq 0 ] || fail_msg "B5: a rejected emit must NOT advance the monotonic seq (got $seq_after)" ) && ok echo "== B6: alarm target resolved BY NAME; beacon.sh inlines no endpoint/secret ==" ( H="$(fresh_home b6)" export ALARM_OUT="$H/alarm.out" export CRED_FILE="$H/credentials.json" export ALARM_TARGET_NAME="primary-monitor" SECRET_TARGET="https://monitor.invalid/alarm/DO-NOT-INLINE-abc123" jq -cn --arg t "$SECRET_TARGET" '{wake:{beacon_alarm_targets:{"primary-monitor":$t}}}' >"$CRED_FILE" export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence -> alarm export WAKE_ALARM_SINK_CMD="$BYNAME_ALARM" out="$("$BEACON" check --slo-seconds 5 2>&1)" rc=$? [ "$rc" -eq 1 ] || fail_msg "B6: absence via a by-name alarm adapter must fire (exit 1); got $rc [$out]" head -n1 "$ALARM_OUT" 2>/dev/null | grep -qF "$SECRET_TARGET" || fail_msg "B6: the adapter must resolve+route to the BY-NAME target [$(cat "$ALARM_OUT" 2>/dev/null)]" # The framework file must NOT inline the endpoint/secret: it only knows a NAME. grep -qF "$SECRET_TARGET" "$BEACON" && fail_msg "B6: beacon.sh must NOT inline the target endpoint/secret" grep -qF "$SECRET_TARGET" "$WAKE_ALARM_SINK_CMD" && fail_msg "B6: even the adapter must resolve by-name, not inline the secret" true ) && ok echo "== B7: unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD ==" ( WAKE_STATE_HOME="$(fresh_home b7)" export WAKE_STATE_HOME export WAKE_BEACON_INDEPENDENCE="off-host" # (a) UNCONFIGURED beacon sink. unset WAKE_BEACON_SINK_CMD out="$("$BEACON" emit 2>&1)" rc=$? [ "$rc" -eq 3 ] || fail_msg "B7a: unconfigured beacon sink must FAIL LOUD (exit 3); got $rc [$out]" echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B7a: the failure must name the silent-no-alarm-host hazard [$out]" # A refused emit (no sink) must not have minted a seq. seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')" [ "${seq_after:-0}" -eq 0 ] || fail_msg "B7a: an unconfigured-sink emit must NOT advance the seq (got $seq_after)" # (b) UNREACHABLE beacon sink (non-zero exit). export WAKE_BEACON_SINK_CMD="false" out2="$("$BEACON" emit 2>&1)" rc2=$? [ "$rc2" -eq 1 ] || fail_msg "B7b: unreachable beacon sink must FAIL LOUD (exit 1); got $rc2 [$out2]" echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B7b: the failure must name the unreachable sink [$out2]" ) && ok echo "== B8: no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) ==" ( H="$(fresh_home b8)" export WAKE_BEACON_RECEIVED="$H/nonexistent.json" export WAKE_ALARM_SINK_CMD="$DISCARD_SINK" out="$("$BEACON" check 2>&1)" rc=$? [ "$rc" -eq 3 ] || fail_msg "B8: check without an SLO must fail loud (exit 3); got $rc [$out]" echo "$out" | grep -qi 'slo' || fail_msg "B8: the usage error must name the missing SLO [$out]" ) && ok echo "== B9: capture-pane hint is a liveness HINT ONLY (does NOT suppress absence) ==" ( H="$(fresh_home b9)" export ALARM_OUT="$H/alarm.json" export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # genuine absence export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" export WAKE_BEACON_PANE_HINT="agent looks idle-at-prompt in capture-pane" out="$("$BEACON" check --slo-seconds 5 2>&1)" rc=$? [ "$rc" -eq 1 ] || fail_msg "B9: a capture-pane hint must NOT suppress a real absence alarm (expected exit 1); got $rc [$out]" [ -s "$ALARM_OUT" ] || fail_msg "B9: absence must still route despite a pane hint (readiness != pane scrape)" ) && ok echo "== B10: fresh beacon within SLO -> ALIVE (exit 0, no alarm) ==" ( H="$(fresh_home b10)" export ALARM_OUT="$H/alarm.json" export WAKE_BEACON_RECEIVED="$H/received.json" export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" now="$(date +%s)" jq -cn --argjson ts "$now" \ '{kind:"wake-beacon", beacon_seq:7, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \ >"$WAKE_BEACON_RECEIVED" out="$("$BEACON" check --slo-seconds 60 2>&1)" rc=$? [ "$rc" -eq 0 ] || fail_msg "B10: a fresh beacon within SLO must be ALIVE (exit 0); got $rc [$out]" echo "$out" | grep -qi 'ALIVE' || fail_msg "B10: a fresh beacon must report ALIVE [$out]" [ ! -s "$ALARM_OUT" ] || fail_msg "B10: a fresh beacon must NOT fire an alarm" ) && ok # ── W7 monitor-integration hardening (folded-in W6 observations, #910 review) ── echo "== B11: staleness from monitor ingested_ts -> a far-future emit_ts STILL goes stale ==" ( H="$(fresh_home b11)" export ALARM_OUT="$H/alarm.json" export WAKE_BEACON_RECEIVED="$H/received.json" export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" now="$(date +%s)" # A host ships a FAR-FUTURE emit_ts (now + 100000s) to try to defer staleness. # record stamps the monitor's own ingested_ts; check must use THAT, not emit_ts. jq -cn --argjson ts "$((now + 100000))" \ '{kind:"wake-beacon", beacon_seq:9, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \ | "$BEACON" record # Simulate the monitor having received it 100s ago (ingested_ts backdated) while # the host's emit_ts stays far in the future. Under emit_ts-based staleness this # would read as ALIVE (age hugely negative); under receive-time it is STALE. jq -c --argjson ing "$((now - 100))" '.ingested_ts = $ing' "$WAKE_BEACON_RECEIVED" >"$H/tmp" && mv "$H/tmp" "$WAKE_BEACON_RECEIVED" out="$("$BEACON" check --slo-seconds 5 2>&1)"; rc=$? [ "$rc" -eq 1 ] || fail_msg "B11: a far-future emit_ts must NOT defer staleness — receive-time governs (expected absence exit 1); got $rc [$out]" echo "$out" | grep -qi 'ALARM FIRED' || fail_msg "B11: the receive-time-stale beacon must fire the absence alarm [$out]" jq -e '.age_seconds >= 100' "$ALARM_OUT" >/dev/null 2>&1 || fail_msg "B11: staleness age must be measured from ingested_ts (>=100s), not emit_ts [$(cat "$ALARM_OUT" 2>/dev/null)]" ) && ok echo "== B12: HMAC-verify at record -> a spoofed (bad-sig) beacon is REJECTED ==" ( H="$(fresh_home b12)" export WAKE_STATE_HOME="$H" export MOSAIC_CREDENTIALS_FILE="$H/credentials.json" export WAKE_BEACON_HMAC_KEY_NAME="beacon" jq -cn --arg k "test-beacon-hmac-key-do-not-echo" '{wake:{hmac_keys:{"beacon":$k}}}' >"$MOSAIC_CREDENTIALS_FILE" export WAKE_BEACON_INDEPENDENCE="off-host" # A capturing sink that keeps the exact shipped (signed) beacon record. SHIPPED="$H/shipped.json" SINK="$H/sink.sh"; printf '#!/usr/bin/env bash\ncat >"%s"\n' "$SHIPPED" >"$SINK"; chmod +x "$SINK" export WAKE_BEACON_SINK_CMD="$SINK" export WAKE_BEACON_RECEIVED="$H/received.json" "$BEACON" emit >/dev/null 2>&1 || fail_msg "B12: a signed emit must succeed with a by-name key" jq -e '.beacon_envelope' "$SHIPPED" >/dev/null 2>&1 || fail_msg "B12: a configured key must produce a signed beacon_envelope [$(cat "$SHIPPED" 2>/dev/null)]" # (a) the authentic signed beacon is ACCEPTED at record. "$BEACON" record <"$SHIPPED" >/dev/null 2>&1 || fail_msg "B12: an authentic signed beacon must be accepted at record" [ -s "$WAKE_BEACON_RECEIVED" ] || fail_msg "B12: the authentic beacon must be stored" # (b) a SPOOFED beacon (fields altered, envelope lifted) is REJECTED. rm -f "$WAKE_BEACON_RECEIVED" spoof="$H/spoof.json" jq -c '.beacon_seq = 99999 | .host_id = "attacker"' "$SHIPPED" >"$spoof" out="$("$BEACON" record <"$spoof" 2>&1)"; rc=$? [ "$rc" -ne 0 ] || fail_msg "B12: a spoofed beacon (altered fields) must be REJECTED at record; got rc=$rc [$out]" echo "$out" | grep -qi 'spoofed beacon' || fail_msg "B12: the rejection must name the spoof [$out]" [ ! -s "$WAKE_BEACON_RECEIVED" ] || fail_msg "B12: a rejected spoof must NOT be stored (dead-man clock not advanced)" # (c) an UNSIGNED beacon is rejected when signing is configured. unsigned="$H/unsigned.json" jq -c 'del(.beacon_envelope)' "$SHIPPED" >"$unsigned" out2="$("$BEACON" record <"$unsigned" 2>&1)"; rc2=$? [ "$rc2" -ne 0 ] || fail_msg "B12: an unsigned beacon must be REJECTED when signing is configured; got rc=$rc2 [$out2]" # beacon.sh must not inline the key material. grep -qF "test-beacon-hmac-key-do-not-echo" "$BEACON" && fail_msg "B12: beacon.sh must NOT inline the HMAC key" true ) && ok echo if [ -s "$FAILFILE" ]; then echo "wake beacon harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 exit 1 fi echo "wake beacon harness: all invariants passed ($pass groups)"