diff --git a/packages/mosaic/framework/tools/wake/beacon.sh b/packages/mosaic/framework/tools/wake/beacon.sh new file mode 100755 index 00000000..4b5d0786 --- /dev/null +++ b/packages/mosaic/framework/tools/wake/beacon.sh @@ -0,0 +1,414 @@ +#!/usr/bin/env bash +# beacon.sh — A8 of the wake canon (EPIC #892, W6): the off-host DEAD-MAN +# liveness BEACON emitter + the pluggable ALARM-SINK adapter + the beacon-ABSENCE +# alarm (the check an off-host monitor runs). +# +# CONTRACT ANCHORS (docs/scratchpads/heartbeat-planning/CONVERGED-DESIGN.md): +# §1.3 Independent liveness leg — off-host dead-man beacon. Liveness is SPLIT +# from work-triggering: the detector emits a MONOTONIC beacon each cycle +# to existing OFF-HOST monitoring, and the alarm fires on beacon +# ABSENCE — depending on NOTHING the dying component must actively do. +# A same-host sibling is CONCEDED NOT independent (shares user-manager, +# host, sender, socket, session — sol A1/A8/G9) and is REJECTED here. A +# single ISOLATED host degrades to a weaker DIFFERENT-SUPERVISION-ROOT +# beacon, FLAGGED as such — never silently pretending full independence. +# `capture-pane` is a liveness HINT ONLY; readiness is proven solely by +# the RECEIVED/CONSUMED acks (W2/W3), never by a pane scrape. +# §4 G1 Off-host dead-man response: the beacon-absence alarm is proven to FIRE +# AND ROUTE to a human/other-host within its SLO. +# §4 G2a Fail-loud semantics: an unconfigured OR unreachable target FAILS LOUD +# (no silent no-alarm host). +# §7-res6 The only real bound on a stuck drainer is an INDEPENDENT +# missing-RECEIVED/CONSUMED escalation through the off-host observer and +# its SLO — the same independent leg this tool provides. +# +# FRAMEWORK / OPERATOR BOUNDARY (§1.4, the operator-agnostic framework/operator split): +# FRAMEWORK (this file) ships: +# - the monotonic beacon EMITTER (`emit`), +# - the off-host monitor's RECEIVER + ABSENCE alarm (`record`, `check`), +# - a pluggable ALARM-SINK / BEACON-SINK ADAPTER INTERFACE. +# OPERATOR owns: +# - the TARGET ENDPOINT (off-host monitor address, alarm route). It is a +# command the operator wires; that command resolves its address/credential +# BY NAME via `load_credentials`, NEVER inlined into this framework file. +# W7 (installer, OUT OF SCOPE here) WIRES + install-validates the target. W6 +# provides the emitter + adapter interface + absence-alarm + the FAIL-LOUD +# PRIMITIVE that W7's validation calls. +# +# ADAPTER INTERFACE (the pluggable seam — operator supplies the command): +# WAKE_BEACON_SINK_CMD emit ships the beacon record (JSON on stdin) via +# `sh -c "$WAKE_BEACON_SINK_CMD"`. The command is the +# transport to the OFF-HOST monitor; it resolves its +# endpoint by-name (load_credentials) internally. A +# non-zero exit = UNREACHABLE target => FAIL LOUD. Unset +# = UNCONFIGURED target => FAIL LOUD (no silent no-alarm +# host). +# WAKE_ALARM_SINK_CMD check routes the absence ALARM (JSON on stdin) via +# `sh -c "$WAKE_ALARM_SINK_CMD"` to a human/other-host +# (G1). Same fail-closed contract: unset OR non-zero +# exit => FAIL LOUD. +# WAKE_BEACON_INDEPENDENCE the operator DECLARES the independence class of the +# wired sink (W7 install-validates it is truthful): +# off-host -> full independence. +# different-supervision-root -> DEGRADED, FLAGGED. +# same-host-sibling -> REJECTED (not independent). +# +# Operator-agnostic: all state via XDG/env; no operator paths/names/secrets/hosts. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./_wake-common.sh disable=SC1091 +. "$SCRIPT_DIR/_wake-common.sh" + +STATE_DIR="$(wake_state_dir)" +# Beacon state (emitter counter + the off-host monitor's received store) lives +# UNDER the wake STATE_DIR in its own subdir so it never collides with the +# store's cursor/inbox files. +BEACON_DIR="$STATE_DIR/beacon" +SEQ_FILE="$BEACON_DIR/seq" +# The off-host monitor's RECEIVED store: the last (highest-seq) beacon this +# monitor has ingested for the host. `check` reads it; `record` writes it. In a +# real deployment this lives on the MONITOR host, fed by the sink transport; the +# default keeps the primitive self-contained + testable. +RECEIVED_FILE="${WAKE_BEACON_RECEIVED:-$BEACON_DIR/received.json}" + +_need_jq() { + command -v jq >/dev/null 2>&1 || { + echo "beacon.sh: jq is required" >&2 + exit 3 + } +} + +usage() { + cat >&2 <<'EOF' +Usage: beacon.sh [options] + +Commands: + emit HOST side. Mint the next MONOTONIC beacon (strictly + increasing seq + emit_ts) and SHIP it off-host via the + pluggable sink adapter (WAKE_BEACON_SINK_CMD). This is + the primitive the detector's run-loop calls each poll. + FAIL-CLOSED: an UNCONFIGURED sink (unset) or an + UNREACHABLE sink (non-zero exit) FAILS LOUD — never a + silent no-alarm host. A same-host-sibling independence + declaration is REJECTED; a different-supervision-root + one is emitted DEGRADED + FLAGGED (never as healthy). + + record OFF-HOST MONITOR side. Ingest a received beacon (JSON + on stdin) into the received store, keeping the highest + seq (a stale/replayed lower-or-equal seq is ignored — + the monotonic contract enforced at the receiver). + + check --slo-seconds N OFF-HOST MONITOR side. The DEAD-MAN / ABSENCE check. + Reads the last received beacon; if it is MISSING or + STALE beyond the SLO, that ABSENCE FIRES the alarm and + ROUTES it via WAKE_ALARM_SINK_CMD to a human/other-host + (G1). Depends on NOTHING the dying host actively does. + Exit 0 = alive (fresh beacon); exit 1 = absence, alarm + fired+routed; exit 3 = fail-closed (missing SLO, or an + unconfigured/unreachable alarm target — loudest). + A capture-pane hint (WAKE_BEACON_PANE_HINT) is a + liveness HINT ONLY and does NOT suppress an absence. + + status Print the emitter counter + last received beacon. + +Options: + --slo-seconds N REQUIRED for check. The absence SLO (operator-tuned; symbolic + per-class tiers per §4 — NO default is invented). A beacon + older than N seconds is an ABSENCE. + +Environment: + WAKE_BEACON_SINK_CMD Pluggable transport to the off-host monitor (emit). + WAKE_ALARM_SINK_CMD Pluggable alarm route to a human/other-host (check). + WAKE_BEACON_INDEPENDENCE off-host | different-supervision-root | + same-host-sibling (REQUIRED for emit; declared by + the operator, install-validated by W7). + WAKE_BEACON_HOST_ID Host identity carried in the beacon (default: hostname). + WAKE_BEACON_SUPERVISION_ROOT Supervision-root label for the degraded beacon. + WAKE_BEACON_RECEIVED Off-host monitor's received-beacon store path. + WAKE_STATE_HOME/WAKE_AGENT wake state namespace (see store.sh). +EOF +} + +# --- monotonic emitter counter (§1.3: a monotonic beacon each cycle) -------- + +# _next_seq — atomically increment + return the beacon counter. Strictly +# increasing across emits (the monotonic contract the off-host monitor relies on +# to distinguish a live-advancing host from a stalled one). +_next_seq() { + mkdir -p "$BEACON_DIR" + local cur nxt + cur="$(_wake_read_int "$SEQ_FILE" 0)" + nxt=$((cur + 1)) + printf '%s' "$nxt" | _atomic_write "$SEQ_FILE" + printf '%s' "$nxt" +} + +# --- independence policy (§1.3: split liveness; reject a non-independent leg) - + +# _resolve_independence — validate the operator's declared independence class and +# echo two space-separated tokens: " ". +# same-host-sibling -> REJECTED (fail loud): shares user-manager/host/ +# sender/socket/session, so it is NOT an independent +# liveness leg and must NEVER stand in for one. +# different-supervision-root -> ACCEPTED but DEGRADED=1: an isolated host with +# no off-host monitor reachable degrades to this +# weaker leg, FLAGGED — never silently "healthy". +# off-host -> ACCEPTED, DEGRADED=0: full independence. +_resolve_independence() { + local decl="${WAKE_BEACON_INDEPENDENCE:-}" + case "$decl" in + off-host) + printf 'off-host 0' + ;; + different-supervision-root) + printf 'different-supervision-root 1' + ;; + same-host-sibling) + echo "beacon.sh: FAIL LOUD (§1.3) — WAKE_BEACON_INDEPENDENCE=same-host-sibling is NOT an independent liveness leg (it shares user-manager/host/sender/socket/session with the component it would supervise). Refusing to emit a beacon that would falsely stand in for off-host supervision." >&2 + return 2 + ;; + '') + echo "beacon.sh: FAIL LOUD — WAKE_BEACON_INDEPENDENCE is unset. Declare the wired sink's independence (off-host | different-supervision-root | same-host-sibling); liveness independence is never silently assumed." >&2 + return 2 + ;; + *) + echo "beacon.sh: FAIL LOUD — WAKE_BEACON_INDEPENDENCE='$decl' is not a recognized class (off-host | different-supervision-root | same-host-sibling)." >&2 + return 2 + ;; + esac +} + +# --- host side: emit ------------------------------------------------------- + +cmd_emit() { + _need_jq + + # Independence FIRST — a same-host-sibling / unknown declaration is rejected + # before any seq is minted, so a rejected emit never advances the counter. + local indep_class degraded parts + parts="$(_resolve_independence)" || exit 2 + indep_class="${parts%% *}" + degraded="${parts##* }" + + # FAIL-CLOSED, part 1 — an UNCONFIGURED off-host target is a silent no-alarm + # host. Refuse before minting a seq (nothing to ship it through). + if [ -z "${WAKE_BEACON_SINK_CMD:-}" ]; then + echo "beacon.sh: FAIL LOUD (G2a) — WAKE_BEACON_SINK_CMD is unset: no off-host beacon target is configured. A host with no beacon sink is a silent no-alarm host. Refusing to no-op." >&2 + exit 3 + fi + + local host_id supervision_root seq emit_ts record + host_id="${WAKE_BEACON_HOST_ID:-$(hostname 2>/dev/null || echo host)}" + supervision_root="${WAKE_BEACON_SUPERVISION_ROOT:-}" + seq="$(_next_seq)" + emit_ts="$(date +%s)" + + record="$(jq -cn \ + --argjson seq "$seq" \ + --argjson emit_ts "$emit_ts" \ + --arg host_id "$host_id" \ + --arg independence "$indep_class" \ + --argjson degraded "$([ "$degraded" -eq 1 ] && echo true || echo false)" \ + --arg supervision_root "$supervision_root" \ + '{kind:"wake-beacon", beacon_seq:$seq, emit_ts:$emit_ts, host_id:$host_id, + independence:$independence, degraded:$degraded} + + (if $supervision_root == "" then {} else {supervision_root:$supervision_root} end)')" + + # DEGRADED beacons are FLAGGED loudly (§1.3) — an isolated host must never + # silently present a weaker different-supervision-root leg as full independence. + if [ "$degraded" -eq 1 ]; then + echo "beacon.sh: DEGRADED beacon (§1.3) — no off-host monitor is reachable; emitting a weaker DIFFERENT-SUPERVISION-ROOT beacon. This is FLAGGED, NOT full independence." >&2 + fi + + # FAIL-CLOSED, part 2 — ship via the pluggable sink adapter. A non-zero exit is + # an UNREACHABLE target => FAIL LOUD (the beacon did not reach off-host). + if ! printf '%s\n' "$record" | sh -c "$WAKE_BEACON_SINK_CMD"; then + echo "beacon.sh: FAIL LOUD (G2a) — beacon sink is UNREACHABLE (WAKE_BEACON_SINK_CMD exited non-zero); beacon seq $seq did NOT reach the off-host monitor." >&2 + exit 1 + fi + + # Local record of the last emit (introspection / the detector seam). The + # AUTHORITATIVE liveness judgement is the off-host monitor's received store, + # never this local copy. + printf '%s\n' "$record" | _atomic_write "$BEACON_DIR/last-emit.json" + printf 'beacon_seq=%s emit_ts=%s independence=%s degraded=%s\n' \ + "$seq" "$emit_ts" "$indep_class" "$([ "$degraded" -eq 1 ] && echo true || echo false)" +} + +# --- off-host monitor side: record a received beacon ----------------------- + +cmd_record() { + _need_jq + local incoming + incoming="$(cat)" + if [ -z "$incoming" ]; then + echo "beacon.sh record: no beacon on stdin" >&2 + exit 2 + fi + local in_seq in_ts + in_seq="$(printf '%s' "$incoming" | jq -r '.beacon_seq // empty' 2>/dev/null)" + in_ts="$(printf '%s' "$incoming" | jq -r '.emit_ts // empty' 2>/dev/null)" + case "$in_seq" in + '' | *[!0-9]*) + echo "beacon.sh record: FAIL LOUD — received beacon has no integer beacon_seq (malformed)" >&2 + exit 2 + ;; + esac + case "$in_ts" in + '' | *[!0-9]*) + echo "beacon.sh record: FAIL LOUD — received beacon has no integer emit_ts (malformed)" >&2 + exit 2 + ;; + esac + + mkdir -p "$(dirname "$RECEIVED_FILE")" + # MONOTONIC receiver: keep the highest-seq beacon. A lower-or-equal seq is a + # stale/replayed beacon and is IGNORED (does not roll the liveness clock back). + local last_seq=0 + if [ -f "$RECEIVED_FILE" ]; then + last_seq="$(jq -r '.beacon_seq // 0' "$RECEIVED_FILE" 2>/dev/null)" + case "$last_seq" in '' | *[!0-9]*) last_seq=0 ;; esac + fi + if [ "$in_seq" -le "$last_seq" ]; then + echo "beacon.sh record: ignoring stale/replayed beacon seq $in_seq (<= recorded $last_seq)" >&2 + exit 0 + fi + printf '%s\n' "$incoming" | _atomic_write "$RECEIVED_FILE" +} + +# --- off-host monitor side: the DEAD-MAN / ABSENCE check ------------------- + +# _fire_alarm REASON PAYLOAD_JSON — route an absence alarm via the pluggable +# alarm-sink adapter to a human/other-host (G1). FAIL-CLOSED: an unconfigured OR +# unreachable alarm target is the loudest failure (a silent no-alarm host is the +# exact condition G1 exists to prevent). +_fire_alarm() { + local reason="$1" payload="$2" + if [ -z "${WAKE_ALARM_SINK_CMD:-}" ]; then + echo "beacon.sh: FAIL LOUD (G1/G2a) — beacon ABSENCE detected ($reason) but WAKE_ALARM_SINK_CMD is UNSET: the alarm cannot route to a human/other-host. This is a silent no-alarm host — the exact failure G1 forbids." >&2 + exit 3 + fi + if ! printf '%s\n' "$payload" | sh -c "$WAKE_ALARM_SINK_CMD"; then + echo "beacon.sh: FAIL LOUD (G1/G2a) — beacon ABSENCE detected ($reason) but the alarm sink is UNREACHABLE (WAKE_ALARM_SINK_CMD exited non-zero): the alarm did NOT reach a human/other-host." >&2 + exit 3 + fi + echo "beacon.sh: ALARM FIRED + ROUTED (§1.3/G1) — $reason" >&2 + exit 1 +} + +cmd_check() { + _need_jq + local slo='' + while [ $# -gt 0 ]; do + case "$1" in + --slo-seconds) + slo="${2:-}" + shift 2 + ;; + *) + echo "beacon.sh check: unknown option '$1'" >&2 + exit 2 + ;; + esac + done + + # No invented SLO (design law) — the per-class absence SLO MUST be supplied. + case "$slo" in + '' | *[!0-9]*) + echo "beacon.sh check: --slo-seconds is REQUIRED and must be a non-negative integer (per-class absence SLO is operator-tuned; no default is invented)." >&2 + exit 3 + ;; + esac + + # capture-pane is a liveness HINT ONLY (§1.3): its presence NEVER suppresses an + # absence. Readiness is proven solely by the received beacon clock (fed by the + # RECEIVED/CONSUMED acks' independent leg), never by a pane scrape. We note the + # hint for operators but do NOT let it gate the dead-man verdict. + if [ -n "${WAKE_BEACON_PANE_HINT:-}" ]; then + echo "beacon.sh check: note — a capture-pane hint is present; it is a liveness HINT ONLY and does not affect this absence verdict." >&2 + fi + + local now + now="$(date +%s)" + + # ABSENCE, case 1 — NEVER received. Depends on nothing the dying host does. + if [ ! -f "$RECEIVED_FILE" ] || [ ! -s "$RECEIVED_FILE" ]; then + local payload + payload="$(jq -cn --argjson now "$now" --argjson slo "$slo" \ + '{kind:"beacon-absence-alarm", reason:"no beacon ever received", + detected_ts:$now, slo_seconds:$slo}')" + _fire_alarm "no beacon ever received" "$payload" + fi + + local last_ts last_seq host_id age + last_ts="$(jq -r '.emit_ts // empty' "$RECEIVED_FILE" 2>/dev/null)" + last_seq="$(jq -r '.beacon_seq // empty' "$RECEIVED_FILE" 2>/dev/null)" + host_id="$(jq -r '.host_id // "unknown"' "$RECEIVED_FILE" 2>/dev/null)" + case "$last_ts" in + '' | *[!0-9]*) + local payload + payload="$(jq -cn --argjson now "$now" --argjson slo "$slo" \ + '{kind:"beacon-absence-alarm", reason:"received beacon has no valid emit_ts (corrupt)", + detected_ts:$now, slo_seconds:$slo}')" + _fire_alarm "corrupt received beacon (no emit_ts)" "$payload" + ;; + esac + + age=$((now - last_ts)) + + # ABSENCE, case 2 — STALE beyond the SLO. A stopped emitter stops advancing the + # received clock; once age > SLO the off-host monitor fires WITHOUT the dying + # host lifting a finger. + if [ "$age" -gt "$slo" ]; then + local payload + payload="$(jq -cn \ + --arg host_id "$host_id" \ + --argjson last_seq "${last_seq:-0}" \ + --argjson last_ts "$last_ts" \ + --argjson now "$now" \ + --argjson age "$age" \ + --argjson slo "$slo" \ + '{kind:"beacon-absence-alarm", reason:"beacon stale past SLO", + host_id:$host_id, last_beacon_seq:$last_seq, last_emit_ts:$last_ts, + detected_ts:$now, age_seconds:$age, slo_seconds:$slo}')" + _fire_alarm "beacon stale (${age}s > SLO ${slo}s), host=$host_id last_seq=${last_seq:-?}" "$payload" + fi + + echo "beacon.sh: ALIVE — last beacon seq ${last_seq:-?} for host '$host_id' is ${age}s old (<= SLO ${slo}s)." +} + +cmd_status() { + local seq + seq="$(_wake_read_int "$SEQ_FILE" 0)" + printf 'beacon_emitter_seq=%s\n' "$seq" + if [ -f "$RECEIVED_FILE" ] && [ -s "$RECEIVED_FILE" ]; then + printf 'last_received=' + cat "$RECEIVED_FILE" + else + printf 'last_received=(none)\n' + fi +} + +main() { + [ $# -ge 1 ] || { + usage + exit 2 + } + local cmd="$1" + shift + case "$cmd" in + emit) cmd_emit "$@" ;; + record) cmd_record "$@" ;; + check) cmd_check "$@" ;; + status) cmd_status "$@" ;; + -h | --help | help) usage ;; + *) + echo "beacon.sh: unknown command '$cmd'" >&2 + usage + exit 2 + ;; + esac +} + +main "$@" diff --git a/packages/mosaic/framework/tools/wake/detector.sh b/packages/mosaic/framework/tools/wake/detector.sh index f7334f36..04344565 100755 --- a/packages/mosaic/framework/tools/wake/detector.sh +++ b/packages/mosaic/framework/tools/wake/detector.sh @@ -402,6 +402,18 @@ cmd_run() { # A single failing source must not kill the long-lived service; poll-once # already failed loud on stderr for it. The loop keeps serving healthy ones. cmd_poll_once || true + # W6 SEAM (§1.3, off-host dead-man beacon): emit ONE monotonic liveness + # beacon per poll cycle. This is the ONLY W6 call site in the detector — a + # single, clean, opt-in seam. It is INERT unless the operator wires a beacon + # sink (WAKE_BEACON_SINK_CMD); wiring + install-validating that target is W7's + # job, not the detector's. Liveness is SPLIT from work-triggering, so a beacon + # emit failure NEVER kills the detector loop — but it is loud (not silent), + # and the off-host monitor's absence check (beacon.sh check) is the real + # safety net regardless of what this dying host does. + if [ -n "${WAKE_BEACON_SINK_CMD:-}" ]; then + "$SCRIPT_DIR/beacon.sh" emit >/dev/null || \ + echo "detector.sh: WARN — off-host liveness beacon emit failed (see beacon.sh); the off-host absence check remains the authoritative dead-man." >&2 + fi [ "$once" -eq 1 ] && break sleep "$interval" done diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index 2be98a9e..64c0644c 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -15,8 +15,10 @@ # 0.2.0 W3 — cumulative-state digest renderer + non-circular HMAC signer. # 0.3.0 W4 — per-host single-instance delta-gated detector daemon. # 0.4.0 W5 — synthetic-canary FN-oracle + source-parity reconciler. +# 0.5.0 W6 — off-host dead-man beacon emitter + pluggable alarm-sink adapter +# + beacon-absence alarm (fail-loud on unconfigured/unreachable). component=wake -version=0.4.0 +version=0.5.0 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version @@ -45,4 +47,16 @@ schema_max=1 # inventory (an omitted source cannot pass the vector # vacuously) + (ii) periodic full reconcile to 0-unaccounted, # enumerating pre-existing/startup state into the store. (W5) -# Out of scope (later waves): off-host beacon (W6), installer (W7). +# beacon.sh A8 — off-host DEAD-MAN liveness beacon: a monotonic beacon +# EMITTER (emit — the primitive the detector run-loop calls +# each cycle), the off-host monitor's RECEIVER + beacon-ABSENCE +# alarm (record, check), and a pluggable alarm-sink/beacon-sink +# ADAPTER INTERFACE. Liveness is SPLIT from work-triggering; +# the alarm fires on ABSENCE, routing to a human/other-host +# within its SLO (§4/G1). FAIL-CLOSED: an unconfigured OR +# unreachable target FAILS LOUD (no silent no-alarm host). +# A same-host sibling is REJECTED as non-independent; an +# isolated host degrades to a FLAGGED different-supervision-root +# beacon; capture-pane is a liveness HINT only. (W6) +# Out of scope (later waves): installer (W7 wires + install-validates the beacon +# target that beacon.sh's fail-loud primitive is designed for). diff --git a/packages/mosaic/framework/tools/wake/test-wake-beacon.sh b/packages/mosaic/framework/tools/wake/test-wake-beacon.sh new file mode 100755 index 00000000..7b854705 --- /dev/null +++ b/packages/mosaic/framework/tools/wake/test-wake-beacon.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# test-wake-beacon.sh — RED-FIRST invariant harness for W6 (EPIC #892): the +# off-host dead-man beacon emitter + pluggable alarm-sink adapter (beacon.sh, A8). +# +# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if +# that invariant regresses: +# B1 beacon emitted each cycle -> MONOTONIC (seq strictly increases) (§1.3) +# B2 beacon ABSENCE past SLO -> alarm FIRES + ROUTES (stopped emitter); +# and NEVER-received -> alarm too — depends on nothing the dying host does (§1.3/G1) +# B3 unconfigured OR unreachable ALARM target -> FAIL LOUD (no silent +# no-alarm host) (§4 G1/G2a) +# B4 isolated host (no off-host monitor) -> DEGRADED different-supervision-root +# beacon FLAGGED, never silently "healthy" (§1.3) +# B5 same-host-sibling declaration -> REJECTED as non-independent, and the +# monotonic seq is NOT advanced by a rejected emit (§1.3) +# B6 the alarm/beacon target is resolved BY NAME by the operator adapter; +# beacon.sh inlines NO endpoint/secret (§1.4) +# B7 unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD (§4 G2a) +# B8 no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) (design law) +# B9 capture-pane hint is a liveness HINT ONLY -> does NOT suppress absence (§1.3) +# B10 fresh beacon within SLO -> ALIVE (exit 0, no alarm) (§1.3) +# +# Uses per-test isolated XDG homes + pluggable sink/alarm adapter SCRIPTS so a +# stopped/dropping emitter and an unreachable target can be exercised. No live +# network, no operator queue touched. +# +# SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell +# and re-exports the per-test env, so environments are isolated by design (the +# same idiom as test-wake-fn-oracle.sh / test-wake-detector.sh). +# shellcheck disable=SC2030,SC2031 +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BEACON="$SCRIPT_DIR/beacon.sh" + +command -v jq >/dev/null 2>&1 || { + echo "SKIP: jq not available" >&2 + exit 0 +} + +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +# Failures recorded to a FILE (subshell-safe — a var counter would silently +# swallow failures across the per-test subshells; mirrors the W2/W4/W5 harnesses). +FAILFILE="$TMP_ROOT/failures" +: >"$FAILFILE" +pass=0 +fail_msg() { + echo " FAIL: $*" >&2 + echo "x" >>"$FAILFILE" +} +ok() { pass=$((pass + 1)); } + +fresh_home() { + local d="$TMP_ROOT/$1" + rm -rf "$d" + mkdir -p "$d" + printf '%s' "$d" +} + +# --- reusable sink/alarm adapter scripts (the pluggable operator seam) ------- + +# discard-sink: a REACHABLE off-host transport that discards (exit 0). +DISCARD_SINK="$TMP_ROOT/discard-sink.sh" +cat >"$DISCARD_SINK" <<'EOF' +#!/usr/bin/env bash +cat >/dev/null +EOF +chmod +x "$DISCARD_SINK" + +# recorder-sink: a REACHABLE transport that feeds the received beacon into the +# off-host monitor's received store via beacon.sh record (WAKE_BEACON_RECEIVED). +RECORDER_SINK="$TMP_ROOT/recorder-sink.sh" +cat >"$RECORDER_SINK" <"$CAPTURE_ALARM" <<'EOF' +#!/usr/bin/env bash +cat >"$ALARM_OUT" +EOF +chmod +x "$CAPTURE_ALARM" + +# byname-alarm: resolves its TARGET endpoint BY NAME from a credential store +# (the load_credentials shape) — never an inline literal — then "delivers" the +# alarm to that resolved target, writing "\n" to \$ALARM_OUT. +BYNAME_ALARM="$TMP_ROOT/byname-alarm.sh" +cat >"$BYNAME_ALARM" <<'EOF' +#!/usr/bin/env bash +set -u +# The operator adapter owns the endpoint: resolved BY NAME from the cred store, +# never passed inline by the framework. +target="$(jq -r --arg n "$ALARM_TARGET_NAME" '.wake.beacon_alarm_targets[$n] // empty' "$CRED_FILE")" +[ -n "$target" ] || { echo "byname-alarm: target name '$ALARM_TARGET_NAME' not found" >&2; exit 1; } +{ printf '%s\n' "$target"; cat; } >"$ALARM_OUT" +EOF +chmod +x "$BYNAME_ALARM" + +echo "== B1: beacon emitted each cycle -> MONOTONIC (seq strictly increases) ==" +( + WAKE_STATE_HOME="$(fresh_home b1)" + export WAKE_STATE_HOME + export WAKE_BEACON_INDEPENDENCE="off-host" + export WAKE_BEACON_SINK_CMD="$DISCARD_SINK" + s1="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')" + s2="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')" + s3="$("$BEACON" emit 2>/dev/null | sed -n 's/.*beacon_seq=\([0-9]*\).*/\1/p')" + [ -n "$s1" ] && [ -n "$s2" ] && [ -n "$s3" ] || fail_msg "B1: emit must print a beacon_seq each cycle [$s1/$s2/$s3]" + { [ "$s2" -gt "$s1" ] && [ "$s3" -gt "$s2" ]; } || fail_msg "B1: beacon seq must STRICTLY increase each cycle (got $s1,$s2,$s3)" +) && ok + +echo "== B2: beacon ABSENCE past SLO -> alarm FIRES + ROUTES ==" +( + H="$(fresh_home b2)" + export ALARM_OUT="$H/alarm.json" + export WAKE_BEACON_RECEIVED="$H/received.json" + export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" + # Simulate a STOPPED emitter: the monitor's last received beacon is old. + now="$(date +%s)" + jq -cn --argjson ts "$((now - 100))" \ + '{kind:"wake-beacon", beacon_seq:5, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \ + >"$WAKE_BEACON_RECEIVED" + out="$("$BEACON" check --slo-seconds 5 2>&1)" + rc=$? + [ "$rc" -eq 1 ] || fail_msg "B2: a stale-past-SLO beacon must FIRE the absence alarm (exit 1); got $rc [$out]" + echo "$out" | grep -qi 'ALARM FIRED' || fail_msg "B2: absence must announce the alarm fired [$out]" + [ -s "$ALARM_OUT" ] || fail_msg "B2: the alarm must actually ROUTE to the sink (payload not written)" + jq -e '.kind == "beacon-absence-alarm"' "$ALARM_OUT" >/dev/null 2>&1 || fail_msg "B2: routed payload must be a beacon-absence-alarm [$(cat "$ALARM_OUT" 2>/dev/null)]" + # NEVER-received is also an absence (depends on nothing the dying host does). + rm -f "$ALARM_OUT" + export WAKE_BEACON_RECEIVED="$H/nonexistent.json" + out2="$("$BEACON" check --slo-seconds 5 2>&1)" + rc2=$? + [ "$rc2" -eq 1 ] || fail_msg "B2: a never-received beacon must FIRE the absence alarm (exit 1); got $rc2 [$out2]" + [ -s "$ALARM_OUT" ] || fail_msg "B2: never-received absence must route to the alarm sink" +) && ok + +echo "== B3: unconfigured OR unreachable ALARM target -> FAIL LOUD ==" +( + H="$(fresh_home b3)" + export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence condition + # (a) UNCONFIGURED alarm sink. + unset WAKE_ALARM_SINK_CMD + out="$("$BEACON" check --slo-seconds 5 2>&1)" + rc=$? + [ "$rc" -eq 3 ] || fail_msg "B3a: unconfigured alarm target must FAIL LOUD (exit 3); got $rc [$out]" + echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B3a: the failure must name the silent-no-alarm-host hazard [$out]" + # (b) UNREACHABLE alarm sink (non-zero exit). + export WAKE_ALARM_SINK_CMD="false" + out2="$("$BEACON" check --slo-seconds 5 2>&1)" + rc2=$? + [ "$rc2" -eq 3 ] || fail_msg "B3b: unreachable alarm target must FAIL LOUD (exit 3); got $rc2 [$out2]" + echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B3b: the failure must name the unreachable target [$out2]" +) && ok + +echo "== B4: isolated host -> DEGRADED different-supervision-root beacon FLAGGED ==" +( + WAKE_STATE_HOME="$(fresh_home b4)" + export WAKE_STATE_HOME + export WAKE_BEACON_INDEPENDENCE="different-supervision-root" + export WAKE_BEACON_SINK_CMD="$DISCARD_SINK" + out="$("$BEACON" emit 2>"$TMP_ROOT/b4.err")" + rc=$? + err="$(cat "$TMP_ROOT/b4.err")" + [ "$rc" -eq 0 ] || fail_msg "B4: a different-supervision-root emit must still succeed (exit 0); got $rc [$out][$err]" + echo "$err" | grep -qi 'DEGRADED' || fail_msg "B4: a different-supervision-root beacon must be FLAGGED degraded on stderr [$err]" + echo "$out" | grep -q 'degraded=true' || fail_msg "B4: emit must NOT silently present a degraded beacon as healthy (degraded=true expected) [$out]" +) && ok + +echo "== B5: same-host-sibling -> REJECTED as non-independent, seq NOT advanced ==" +( + WAKE_STATE_HOME="$(fresh_home b5)" + export WAKE_STATE_HOME + export WAKE_BEACON_INDEPENDENCE="same-host-sibling" + export WAKE_BEACON_SINK_CMD="$DISCARD_SINK" + out="$("$BEACON" emit 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "B5: a same-host-sibling beacon must be REJECTED (non-zero exit) [$out]" + echo "$out" | grep -qi 'not an independent' || fail_msg "B5: the rejection must explain it is NOT an independent leg [$out]" + # A rejected emit must not have minted a seq (rejection precedes counter bump). + seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')" + [ "${seq_after:-0}" -eq 0 ] || fail_msg "B5: a rejected emit must NOT advance the monotonic seq (got $seq_after)" +) && ok + +echo "== B6: alarm target resolved BY NAME; beacon.sh inlines no endpoint/secret ==" +( + H="$(fresh_home b6)" + export ALARM_OUT="$H/alarm.out" + export CRED_FILE="$H/credentials.json" + export ALARM_TARGET_NAME="primary-monitor" + SECRET_TARGET="https://monitor.invalid/alarm/DO-NOT-INLINE-abc123" + jq -cn --arg t "$SECRET_TARGET" '{wake:{beacon_alarm_targets:{"primary-monitor":$t}}}' >"$CRED_FILE" + export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # absence -> alarm + export WAKE_ALARM_SINK_CMD="$BYNAME_ALARM" + out="$("$BEACON" check --slo-seconds 5 2>&1)" + rc=$? + [ "$rc" -eq 1 ] || fail_msg "B6: absence via a by-name alarm adapter must fire (exit 1); got $rc [$out]" + head -n1 "$ALARM_OUT" 2>/dev/null | grep -qF "$SECRET_TARGET" || fail_msg "B6: the adapter must resolve+route to the BY-NAME target [$(cat "$ALARM_OUT" 2>/dev/null)]" + # The framework file must NOT inline the endpoint/secret: it only knows a NAME. + grep -qF "$SECRET_TARGET" "$BEACON" && fail_msg "B6: beacon.sh must NOT inline the target endpoint/secret" + grep -qF "$SECRET_TARGET" "$WAKE_ALARM_SINK_CMD" && fail_msg "B6: even the adapter must resolve by-name, not inline the secret" + true +) && ok + +echo "== B7: unconfigured OR unreachable BEACON sink on emit -> FAIL LOUD ==" +( + WAKE_STATE_HOME="$(fresh_home b7)" + export WAKE_STATE_HOME + export WAKE_BEACON_INDEPENDENCE="off-host" + # (a) UNCONFIGURED beacon sink. + unset WAKE_BEACON_SINK_CMD + out="$("$BEACON" emit 2>&1)" + rc=$? + [ "$rc" -eq 3 ] || fail_msg "B7a: unconfigured beacon sink must FAIL LOUD (exit 3); got $rc [$out]" + echo "$out" | grep -qi 'silent no-alarm host' || fail_msg "B7a: the failure must name the silent-no-alarm-host hazard [$out]" + # A refused emit (no sink) must not have minted a seq. + seq_after="$("$BEACON" status 2>/dev/null | sed -n 's/^beacon_emitter_seq=//p')" + [ "${seq_after:-0}" -eq 0 ] || fail_msg "B7a: an unconfigured-sink emit must NOT advance the seq (got $seq_after)" + # (b) UNREACHABLE beacon sink (non-zero exit). + export WAKE_BEACON_SINK_CMD="false" + out2="$("$BEACON" emit 2>&1)" + rc2=$? + [ "$rc2" -eq 1 ] || fail_msg "B7b: unreachable beacon sink must FAIL LOUD (exit 1); got $rc2 [$out2]" + echo "$out2" | grep -qi 'UNREACHABLE' || fail_msg "B7b: the failure must name the unreachable sink [$out2]" +) && ok + +echo "== B8: no invented SLO -> check --slo-seconds is REQUIRED (fail-loud) ==" +( + H="$(fresh_home b8)" + export WAKE_BEACON_RECEIVED="$H/nonexistent.json" + export WAKE_ALARM_SINK_CMD="$DISCARD_SINK" + out="$("$BEACON" check 2>&1)" + rc=$? + [ "$rc" -eq 3 ] || fail_msg "B8: check without an SLO must fail loud (exit 3); got $rc [$out]" + echo "$out" | grep -qi 'slo' || fail_msg "B8: the usage error must name the missing SLO [$out]" +) && ok + +echo "== B9: capture-pane hint is a liveness HINT ONLY (does NOT suppress absence) ==" +( + H="$(fresh_home b9)" + export ALARM_OUT="$H/alarm.json" + export WAKE_BEACON_RECEIVED="$H/nonexistent.json" # genuine absence + export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" + export WAKE_BEACON_PANE_HINT="agent looks idle-at-prompt in capture-pane" + out="$("$BEACON" check --slo-seconds 5 2>&1)" + rc=$? + [ "$rc" -eq 1 ] || fail_msg "B9: a capture-pane hint must NOT suppress a real absence alarm (expected exit 1); got $rc [$out]" + [ -s "$ALARM_OUT" ] || fail_msg "B9: absence must still route despite a pane hint (readiness != pane scrape)" +) && ok + +echo "== B10: fresh beacon within SLO -> ALIVE (exit 0, no alarm) ==" +( + H="$(fresh_home b10)" + export ALARM_OUT="$H/alarm.json" + export WAKE_BEACON_RECEIVED="$H/received.json" + export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" + now="$(date +%s)" + jq -cn --argjson ts "$now" \ + '{kind:"wake-beacon", beacon_seq:7, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \ + >"$WAKE_BEACON_RECEIVED" + out="$("$BEACON" check --slo-seconds 60 2>&1)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "B10: a fresh beacon within SLO must be ALIVE (exit 0); got $rc [$out]" + echo "$out" | grep -qi 'ALIVE' || fail_msg "B10: a fresh beacon must report ALIVE [$out]" + [ ! -s "$ALARM_OUT" ] || fail_msg "B10: a fresh beacon must NOT fire an alarm" +) && ok + +echo +if [ -s "$FAILFILE" ]; then + echo "wake beacon harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 + exit 1 +fi +echo "wake beacon harness: all invariants passed ($pass groups)" diff --git a/packages/mosaic/package.json b/packages/mosaic/package.json index 767ff66d..8dc6a4b1 100644 --- a/packages/mosaic/package.json +++ b/packages/mosaic/package.json @@ -25,7 +25,7 @@ "lint": "eslint src", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests && pnpm run test:framework-shell", - "test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh" + "test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh" }, "dependencies": { "@mosaicstack/brain": "workspace:*",