Files
stack/packages/mosaic/framework/tools/wake/test-wake-beacon.sh
jason.woltje 2378665eaf
All checks were successful
ci/woodpecker/push/publish Pipeline was successful
ci/woodpecker/push/ci Pipeline was successful
feat(wake): W7 A10 idempotent installer + mosaic-wake.service (component-manifest, Gate-A, blank-reset retire, snapshot-guard, fail-closed install-validate) (#911)
Co-authored-by: jason.woltje <jason@diversecanvas.com>
Co-committed-by: jason.woltje <jason@diversecanvas.com>
2026-07-26 04:19:09 +00:00

344 lines
17 KiB
Bash
Executable File

#!/usr/bin/env bash
# test-wake-beacon.sh — RED-FIRST invariant harness for W6 (EPIC #892): the
# off-host dead-man beacon emitter + pluggable alarm-sink adapter (beacon.sh, A8).
#
# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if
# that invariant regresses:
# B1 beacon emitted each cycle -> MONOTONIC (seq strictly increases) (§1.3)
# B2 beacon ABSENCE past SLO -> alarm FIRES + ROUTES (stopped emitter);
# and NEVER-received -> alarm too — depends on nothing the dying host does (§1.3/G1)
# B3 unconfigured OR unreachable ALARM target -> FAIL LOUD (no silent
# no-alarm host) (§4 G1/G2a)
# B4 isolated host (no off-host monitor) -> DEGRADED different-supervision-root
# beacon FLAGGED, never silently "healthy" (§1.3)
# B5 same-host-sibling declaration -> REJECTED as non-independent, and the
# monotonic seq is NOT advanced by a rejected emit (§1.3)
# B6 the alarm/beacon target is resolved BY NAME by the operator adapter;
# beacon.sh inlines NO endpoint/secret (§1.4)
# B7 unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD (§4 G2a)
# B8 no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) (design law)
# B9 capture-pane hint is a liveness HINT ONLY -> does NOT suppress absence (§1.3)
# B10 fresh beacon within SLO -> ALIVE (exit 0, no alarm) (§1.3)
#
# Uses per-test isolated XDG homes + pluggable sink/alarm adapter SCRIPTS so a
# stopped/dropping emitter and an unreachable target can be exercised. No live
# network, no operator queue touched.
#
# SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell
# and re-exports the per-test env, so environments are isolated by design (the
# same idiom as test-wake-fn-oracle.sh / test-wake-detector.sh).
# shellcheck disable=SC2030,SC2031
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BEACON="$SCRIPT_DIR/beacon.sh"
command -v jq >/dev/null 2>&1 || {
echo "SKIP: jq not available" >&2
exit 0
}
TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMP_ROOT"' EXIT
# Failures recorded to a FILE (subshell-safe — a var counter would silently
# swallow failures across the per-test subshells; mirrors the W2/W4/W5 harnesses).
FAILFILE="$TMP_ROOT/failures"
: >"$FAILFILE"
pass=0
fail_msg() {
echo " FAIL: $*" >&2
echo "x" >>"$FAILFILE"
}
ok() { pass=$((pass + 1)); }
fresh_home() {
local d="$TMP_ROOT/$1"
rm -rf "$d"
mkdir -p "$d"
printf '%s' "$d"
}
# --- reusable sink/alarm adapter scripts (the pluggable operator seam) -------
# discard-sink: a REACHABLE off-host transport that discards (exit 0).
DISCARD_SINK="$TMP_ROOT/discard-sink.sh"
cat >"$DISCARD_SINK" <<'EOF'
#!/usr/bin/env bash
cat >/dev/null
EOF
chmod +x "$DISCARD_SINK"
# recorder-sink: a REACHABLE transport that feeds the received beacon into the
# off-host monitor's received store via beacon.sh record (WAKE_BEACON_RECEIVED).
RECORDER_SINK="$TMP_ROOT/recorder-sink.sh"
cat >"$RECORDER_SINK" <<EOF
#!/usr/bin/env bash
exec "$BEACON" record
EOF
chmod +x "$RECORDER_SINK"
# capture-alarm: a REACHABLE alarm route that writes the routed payload to
# \$ALARM_OUT (so a test can prove the alarm actually reached the sink).
CAPTURE_ALARM="$TMP_ROOT/capture-alarm.sh"
cat >"$CAPTURE_ALARM" <<'EOF'
#!/usr/bin/env bash
cat >"$ALARM_OUT"
EOF
chmod +x "$CAPTURE_ALARM"
# byname-alarm: resolves its TARGET endpoint BY NAME from a credential store
# (the load_credentials shape) — never an inline literal — then "delivers" the
# alarm to that resolved target, writing "<target>\n<payload>" to \$ALARM_OUT.
BYNAME_ALARM="$TMP_ROOT/byname-alarm.sh"
cat >"$BYNAME_ALARM" <<'EOF'
#!/usr/bin/env bash
set -u
# The operator adapter owns the endpoint: resolved BY NAME from the cred store,
# never passed inline by the framework.
target="$(jq -r --arg n "$ALARM_TARGET_NAME" '.wake.beacon_alarm_targets[$n] // empty' "$CRED_FILE")"
[ -n "$target" ] || { echo "byname-alarm: target name '$ALARM_TARGET_NAME' not found" >&2; exit 1; }
{ printf '%s\n' "$target"; cat; } >"$ALARM_OUT"
EOF
chmod +x "$BYNAME_ALARM"
echo "== B1: beacon emitted each cycle -> MONOTONIC (seq strictly increases) =="
(
WAKE_STATE_HOME="$(fresh_home b1)"
export WAKE_STATE_HOME
export WAKE_BEACON_INDEPENDENCE="off-host"
export WAKE_BEACON_SINK_CMD="$DISCARD_SINK"
s1="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')"
s2="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')"
s3="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')"
[ -n "$s1" ] && [ -n "$s2" ] && [ -n "$s3" ] || fail_msg "B1: emit must print a beacon_seq each cycle [$s1/$s2/$s3]"
{ [ "$s2" -gt "$s1" ] && [ "$s3" -gt "$s2" ]; } || fail_msg "B1: beacon seq must STRICTLY increase each cycle (got $s1,$s2,$s3)"
) && ok
echo "== B2: beacon ABSENCE past SLO -> alarm FIRES + ROUTES =="
(
H="$(fresh_home b2)"
export ALARM_OUT="$H/alarm.json"
export WAKE_BEACON_RECEIVED="$H/received.json"
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
# Simulate a STOPPED emitter: the monitor's last received beacon is old.
now="$(date +%s)"
jq -cn --argjson ts "$((now - 100))" \
'{kind:"wake-beacon", beacon_seq:5, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \
>"$WAKE_BEACON_RECEIVED"
out="$("$BEACON" check --slo-seconds 5 2>&1)"
rc=$?
[ "$rc" -eq 1 ] || fail_msg "B2: a stale-past-SLO beacon must FIRE the absence alarm (exit 1); got $rc [$out]"
echo "$out" | grep -qi 'ALARM FIRED' || fail_msg "B2: absence must announce the alarm fired [$out]"
[ -s "$ALARM_OUT" ] || fail_msg "B2: the alarm must actually ROUTE to the sink (payload not written)"
jq -e '.kind == "beacon-absence-alarm"' "$ALARM_OUT" >/dev/null 2>&1 || fail_msg "B2: routed payload must be a beacon-absence-alarm [$(cat "$ALARM_OUT" 2>/dev/null)]"
# NEVER-received is also an absence (depends on nothing the dying host does).
rm -f "$ALARM_OUT"
export WAKE_BEACON_RECEIVED="$H/nonexistent.json"
out2="$("$BEACON" check --slo-seconds 5 2>&1)"
rc2=$?
[ "$rc2" -eq 1 ] || fail_msg "B2: a never-received beacon must FIRE the absence alarm (exit 1); got $rc2 [$out2]"
[ -s "$ALARM_OUT" ] || fail_msg "B2: never-received absence must route to the alarm sink"
) && ok
echo "== B3: unconfigured OR unreachable ALARM target -> FAIL LOUD =="
(
H="$(fresh_home b3)"
export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence condition
# (a) UNCONFIGURED alarm sink.
unset WAKE_ALARM_SINK_CMD
out="$("$BEACON" check --slo-seconds 5 2>&1)"
rc=$?
[ "$rc" -eq 3 ] || fail_msg "B3a: unconfigured alarm target must FAIL LOUD (exit 3); got $rc [$out]"
echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B3a: the failure must name the silent-no-alarm-host hazard [$out]"
# (b) UNREACHABLE alarm sink (non-zero exit).
export WAKE_ALARM_SINK_CMD="false"
out2="$("$BEACON" check --slo-seconds 5 2>&1)"
rc2=$?
[ "$rc2" -eq 3 ] || fail_msg "B3b: unreachable alarm target must FAIL LOUD (exit 3); got $rc2 [$out2]"
echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B3b: the failure must name the unreachable target [$out2]"
) && ok
echo "== B4: isolated host -> DEGRADED different-supervision-root beacon FLAGGED =="
(
WAKE_STATE_HOME="$(fresh_home b4)"
export WAKE_STATE_HOME
export WAKE_BEACON_INDEPENDENCE="different-supervision-root"
export WAKE_BEACON_SINK_CMD="$DISCARD_SINK"
out="$("$BEACON" emit 2>"$TMP_ROOT/b4.err")"
rc=$?
err="$(cat "$TMP_ROOT/b4.err")"
[ "$rc" -eq 0 ] || fail_msg "B4: a different-supervision-root emit must still succeed (exit 0); got $rc [$out][$err]"
echo "$err" | grep -qi 'DEGRADED' || fail_msg "B4: a different-supervision-root beacon must be FLAGGED degraded on stderr [$err]"
echo "$out" | grep -q 'degraded=true' || fail_msg "B4: emit must NOT silently present a degraded beacon as healthy (degraded=true expected) [$out]"
) && ok
echo "== B5: same-host-sibling -> REJECTED as non-independent, seq NOT advanced =="
(
WAKE_STATE_HOME="$(fresh_home b5)"
export WAKE_STATE_HOME
export WAKE_BEACON_INDEPENDENCE="same-host-sibling"
export WAKE_BEACON_SINK_CMD="$DISCARD_SINK"
out="$("$BEACON" emit 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "B5: a same-host-sibling beacon must be REJECTED (non-zero exit) [$out]"
echo "$out" | grep -qi 'not an independent' || fail_msg "B5: the rejection must explain it is NOT an independent leg [$out]"
# A rejected emit must not have minted a seq (rejection precedes counter bump).
seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')"
[ "${seq_after:-0}" -eq 0 ] || fail_msg "B5: a rejected emit must NOT advance the monotonic seq (got $seq_after)"
) && ok
echo "== B6: alarm target resolved BY NAME; beacon.sh inlines no endpoint/secret =="
(
H="$(fresh_home b6)"
export ALARM_OUT="$H/alarm.out"
export CRED_FILE="$H/credentials.json"
export ALARM_TARGET_NAME="primary-monitor"
SECRET_TARGET="https://monitor.invalid/alarm/DO-NOT-INLINE-abc123"
jq -cn --arg t "$SECRET_TARGET" '{wake:{beacon_alarm_targets:{"primary-monitor":$t}}}' >"$CRED_FILE"
export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence -> alarm
export WAKE_ALARM_SINK_CMD="$BYNAME_ALARM"
out="$("$BEACON" check --slo-seconds 5 2>&1)"
rc=$?
[ "$rc" -eq 1 ] || fail_msg "B6: absence via a by-name alarm adapter must fire (exit 1); got $rc [$out]"
head -n1 "$ALARM_OUT" 2>/dev/null | grep -qF "$SECRET_TARGET" || fail_msg "B6: the adapter must resolve+route to the BY-NAME target [$(cat "$ALARM_OUT" 2>/dev/null)]"
# The framework file must NOT inline the endpoint/secret: it only knows a NAME.
grep -qF "$SECRET_TARGET" "$BEACON" && fail_msg "B6: beacon.sh must NOT inline the target endpoint/secret"
grep -qF "$SECRET_TARGET" "$WAKE_ALARM_SINK_CMD" && fail_msg "B6: even the adapter must resolve by-name, not inline the secret"
true
) && ok
echo "== B7: unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD =="
(
WAKE_STATE_HOME="$(fresh_home b7)"
export WAKE_STATE_HOME
export WAKE_BEACON_INDEPENDENCE="off-host"
# (a) UNCONFIGURED beacon sink.
unset WAKE_BEACON_SINK_CMD
out="$("$BEACON" emit 2>&1)"
rc=$?
[ "$rc" -eq 3 ] || fail_msg "B7a: unconfigured beacon sink must FAIL LOUD (exit 3); got $rc [$out]"
echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B7a: the failure must name the silent-no-alarm-host hazard [$out]"
# A refused emit (no sink) must not have minted a seq.
seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')"
[ "${seq_after:-0}" -eq 0 ] || fail_msg "B7a: an unconfigured-sink emit must NOT advance the seq (got $seq_after)"
# (b) UNREACHABLE beacon sink (non-zero exit).
export WAKE_BEACON_SINK_CMD="false"
out2="$("$BEACON" emit 2>&1)"
rc2=$?
[ "$rc2" -eq 1 ] || fail_msg "B7b: unreachable beacon sink must FAIL LOUD (exit 1); got $rc2 [$out2]"
echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B7b: the failure must name the unreachable sink [$out2]"
) && ok
echo "== B8: no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) =="
(
H="$(fresh_home b8)"
export WAKE_BEACON_RECEIVED="$H/nonexistent.json"
export WAKE_ALARM_SINK_CMD="$DISCARD_SINK"
out="$("$BEACON" check 2>&1)"
rc=$?
[ "$rc" -eq 3 ] || fail_msg "B8: check without an SLO must fail loud (exit 3); got $rc [$out]"
echo "$out" | grep -qi 'slo' || fail_msg "B8: the usage error must name the missing SLO [$out]"
) && ok
echo "== B9: capture-pane hint is a liveness HINT ONLY (does NOT suppress absence) =="
(
H="$(fresh_home b9)"
export ALARM_OUT="$H/alarm.json"
export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # genuine absence
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
export WAKE_BEACON_PANE_HINT="agent looks idle-at-prompt in capture-pane"
out="$("$BEACON" check --slo-seconds 5 2>&1)"
rc=$?
[ "$rc" -eq 1 ] || fail_msg "B9: a capture-pane hint must NOT suppress a real absence alarm (expected exit 1); got $rc [$out]"
[ -s "$ALARM_OUT" ] || fail_msg "B9: absence must still route despite a pane hint (readiness != pane scrape)"
) && ok
echo "== B10: fresh beacon within SLO -> ALIVE (exit 0, no alarm) =="
(
H="$(fresh_home b10)"
export ALARM_OUT="$H/alarm.json"
export WAKE_BEACON_RECEIVED="$H/received.json"
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
now="$(date +%s)"
jq -cn --argjson ts "$now" \
'{kind:"wake-beacon", beacon_seq:7, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \
>"$WAKE_BEACON_RECEIVED"
out="$("$BEACON" check --slo-seconds 60 2>&1)"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "B10: a fresh beacon within SLO must be ALIVE (exit 0); got $rc [$out]"
echo "$out" | grep -qi 'ALIVE' || fail_msg "B10: a fresh beacon must report ALIVE [$out]"
[ ! -s "$ALARM_OUT" ] || fail_msg "B10: a fresh beacon must NOT fire an alarm"
) && ok
# ── W7 monitor-integration hardening (folded-in W6 observations, #910 review) ──
echo "== B11: staleness from monitor ingested_ts -> a far-future emit_ts STILL goes stale =="
(
H="$(fresh_home b11)"
export ALARM_OUT="$H/alarm.json"
export WAKE_BEACON_RECEIVED="$H/received.json"
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
now="$(date +%s)"
# A host ships a FAR-FUTURE emit_ts (now + 100000s) to try to defer staleness.
# record stamps the monitor's own ingested_ts; check must use THAT, not emit_ts.
jq -cn --argjson ts "$((now + 100000))" \
'{kind:"wake-beacon", beacon_seq:9, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \
| "$BEACON" record
# Simulate the monitor having received it 100s ago (ingested_ts backdated) while
# the host's emit_ts stays far in the future. Under emit_ts-based staleness this
# would read as ALIVE (age hugely negative); under receive-time it is STALE.
jq -c --argjson ing "$((now - 100))" '.ingested_ts = $ing' "$WAKE_BEACON_RECEIVED" >"$H/tmp" && mv "$H/tmp" "$WAKE_BEACON_RECEIVED"
out="$("$BEACON" check --slo-seconds 5 2>&1)"; rc=$?
[ "$rc" -eq 1 ] || fail_msg "B11: a far-future emit_ts must NOT defer staleness — receive-time governs (expected absence exit 1); got $rc [$out]"
echo "$out" | grep -qi 'ALARM FIRED' || fail_msg "B11: the receive-time-stale beacon must fire the absence alarm [$out]"
jq -e '.age_seconds >= 100' "$ALARM_OUT" >/dev/null 2>&1 || fail_msg "B11: staleness age must be measured from ingested_ts (>=100s), not emit_ts [$(cat "$ALARM_OUT" 2>/dev/null)]"
) && ok
echo "== B12: HMAC-verify at record -> a spoofed (bad-sig) beacon is REJECTED =="
if ! command -v openssl >/dev/null 2>&1; then
echo "SKIP: openssl not available" >&2
else
(
H="$(fresh_home b12)"
export WAKE_STATE_HOME="$H"
export MOSAIC_CREDENTIALS_FILE="$H/credentials.json"
export WAKE_BEACON_HMAC_KEY_NAME="beacon"
jq -cn --arg k "test-beacon-hmac-key-do-not-echo" '{wake:{hmac_keys:{"beacon":$k}}}' >"$MOSAIC_CREDENTIALS_FILE"
export WAKE_BEACON_INDEPENDENCE="off-host"
# A capturing sink that keeps the exact shipped (signed) beacon record.
SHIPPED="$H/shipped.json"
SINK="$H/sink.sh"; printf '#!/usr/bin/env bash\ncat >"%s"\n' "$SHIPPED" >"$SINK"; chmod +x "$SINK"
export WAKE_BEACON_SINK_CMD="$SINK"
export WAKE_BEACON_RECEIVED="$H/received.json"
"$BEACON" emit >/dev/null 2>&1 || fail_msg "B12: a signed emit must succeed with a by-name key"
jq -e '.beacon_envelope' "$SHIPPED" >/dev/null 2>&1 || fail_msg "B12: a configured key must produce a signed beacon_envelope [$(cat "$SHIPPED" 2>/dev/null)]"
# (a) the authentic signed beacon is ACCEPTED at record.
"$BEACON" record <"$SHIPPED" >/dev/null 2>&1 || fail_msg "B12: an authentic signed beacon must be accepted at record"
[ -s "$WAKE_BEACON_RECEIVED" ] || fail_msg "B12: the authentic beacon must be stored"
# (b) a SPOOFED beacon (fields altered, envelope lifted) is REJECTED.
rm -f "$WAKE_BEACON_RECEIVED"
spoof="$H/spoof.json"
jq -c '.beacon_seq = 99999 | .host_id = "attacker"' "$SHIPPED" >"$spoof"
out="$("$BEACON" record <"$spoof" 2>&1)"; rc=$?
[ "$rc" -ne 0 ] || fail_msg "B12: a spoofed beacon (altered fields) must be REJECTED at record; got rc=$rc [$out]"
echo "$out" | grep -qi 'spoofed beacon' || fail_msg "B12: the rejection must name the spoof [$out]"
[ ! -s "$WAKE_BEACON_RECEIVED" ] || fail_msg "B12: a rejected spoof must NOT be stored (dead-man clock not advanced)"
# (c) an UNSIGNED beacon is rejected when signing is configured.
unsigned="$H/unsigned.json"
jq -c 'del(.beacon_envelope)' "$SHIPPED" >"$unsigned"
out2="$("$BEACON" record <"$unsigned" 2>&1)"; rc2=$?
[ "$rc2" -ne 0 ] || fail_msg "B12: an unsigned beacon must be REJECTED when signing is configured; got rc=$rc2 [$out2]"
# beacon.sh must not inline the key material.
grep -qF "test-beacon-hmac-key-do-not-echo" "$BEACON" && fail_msg "B12: beacon.sh must NOT inline the HMAC key"
true
) && ok
fi
echo
if [ -s "$FAILFILE" ]; then
echo "wake beacon harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
exit 1
fi
echo "wake beacon harness: all invariants passed ($pass groups)"