jason.woltje
712c770b7a
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
257 lines
13 KiB
Bash
Executable File
257 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-wake-install.sh — RED-FIRST invariant harness for W7 (EPIC #892, A10): the
|
|
# idempotent, fail-closed wake COMPONENT INSTALLER (wake-install.sh) + the W6
|
|
# monitor-integration hardening in beacon.sh (folded-in observation set).
|
|
#
|
|
# Each group asserts ONE enforcement-path invariant and is designed to go RED if
|
|
# that invariant regresses (a targeted mutation to wake-install.sh / beacon.sh
|
|
# makes exactly its group fail):
|
|
# I1 idempotency — a second component install produces NO diff / no rewrite (i)
|
|
# I2 Gate A — a candidate the framework-manifest SSOT does not own is REFUSED,
|
|
# fail-closed, with NO partial write (the wake manifest authorizes nothing) (i)
|
|
# I3 blank-reset — the reset-line idiom collapses the cadence to EXACTLY ONE
|
|
# OnUnitActiveUSec; without it, two values leak (negative control) (iii)
|
|
# I4 snapshot-guard — a reap with NO prior snapshot is REFUSED (fail-closed);
|
|
# after a snapshot it is allowed (iv)
|
|
# I5 fail-closed install-validate — unconfigured HMAC key OR unconfigured/
|
|
# unreachable alarm target FAILS LOUD (non-zero); configured passes; and
|
|
# wake-install.sh inlines NO secret/endpoint (v)
|
|
# I6 reset->verify->retire lifecycle — overlap leaves the legacy timer running;
|
|
# only a §4-vector pass retires it, and only snapshot-guarded (iii)
|
|
# I7 ingested_ts staleness — staleness is computed from the monitor's
|
|
# receive-time, so a host shipping a FAR-FUTURE emit_ts STILL goes stale (vi-a)
|
|
# I8 beacon HMAC-verify at record — a spoofed (bad-sig) beacon is REJECTED (vi-b)
|
|
# (I7/I8 exercise the W6 monitor-integration hardening folded into beacon.sh; the
|
|
# same invariants are also asserted in depth by test-wake-beacon.sh B11/B12.)
|
|
#
|
|
# Isolated per-group XDG/systemd/target dirs; no live network, no operator queue,
|
|
# no real systemd manager touched (the blank-reset verify uses the deterministic
|
|
# merge simulation, which mirrors `systemctl show -p OnUnitActiveUSec`).
|
|
#
|
|
# SC2030/SC2031 disabled: each group runs in its own ( ) subshell and re-exports
|
|
# its env, so environments are isolated by design (same idiom as the W2-W6 harnesses).
|
|
# shellcheck disable=SC2030,SC2031
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
WI="$SCRIPT_DIR/wake-install.sh"
|
|
BEACON="$SCRIPT_DIR/beacon.sh"
|
|
FRAMEWORK_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
command -v jq >/dev/null 2>&1 || { echo "SKIP: jq not available" >&2; exit 0; }
|
|
|
|
TMP_ROOT="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP_ROOT"' EXIT
|
|
|
|
FAILFILE="$TMP_ROOT/failures"
|
|
: >"$FAILFILE"
|
|
pass=0
|
|
fail_msg() { echo " FAIL: $*" >&2; echo "x" >>"$FAILFILE"; }
|
|
ok() { pass=$((pass + 1)); }
|
|
|
|
fresh() { local d="$TMP_ROOT/$1"; rm -rf "$d"; mkdir -p "$d"; printf '%s' "$d"; }
|
|
|
|
echo "== I1: idempotency — a second component install produces NO diff =="
|
|
(
|
|
TGT="$(fresh i1-target)"
|
|
export WAKE_INSTALL_SOURCE="$FRAMEWORK_ROOT"
|
|
export WAKE_INSTALL_TARGET="$TGT"
|
|
out1="$(bash "$WI" install 2>/dev/null)"
|
|
w1="$(echo "$out1" | sed -n 's/.*written=\([0-9]*\).*/\1/p')"
|
|
[ "${w1:-0}" -gt 0 ] || fail_msg "I1: first install must write the wake component (written=$w1)"
|
|
# Snapshot the installed tree, run again, and require byte-identical output.
|
|
cp1="$(fresh i1-copy)"; cp -a "$TGT/." "$cp1/"
|
|
out2="$(bash "$WI" install 2>/dev/null)"
|
|
w2="$(echo "$out2" | sed -n 's/.*written=\([0-9]*\).*/\1/p')"
|
|
[ "${w2:-1}" -eq 0 ] || fail_msg "I1: a re-run must write ZERO files (idempotent); got written=$w2"
|
|
if ! diff -r "$cp1" "$TGT" >/dev/null 2>&1; then
|
|
fail_msg "I1: second install changed the target tree (not idempotent)"
|
|
fi
|
|
) && ok
|
|
|
|
echo "== I2: Gate A — a non-framework-owned candidate is REFUSED (fail-closed, no partial write) =="
|
|
(
|
|
TGT="$(fresh i2-target)"
|
|
# A manifest that DE-AUTHORIZES the wake tools: tools/wake/** is operator-owned.
|
|
# The install MUST refuse the whole thing and write nothing (deny-wins).
|
|
BAD_MANIFEST="$TMP_ROOT/i2-manifest.txt"
|
|
cat >"$BAD_MANIFEST" <<'EOF'
|
|
[framework]
|
|
tools/**
|
|
systemd/**
|
|
defaults/**
|
|
[operator]
|
|
tools/wake/**
|
|
EOF
|
|
export WAKE_INSTALL_SOURCE="$FRAMEWORK_ROOT"
|
|
export WAKE_INSTALL_TARGET="$TGT"
|
|
export WAKE_INSTALL_MANIFEST="$BAD_MANIFEST"
|
|
out="$(bash "$WI" install 2>&1)"; rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "I2: install must FAIL when a wake candidate is not framework-owned (got rc=$rc)"
|
|
echo "$out" | grep -qi 'Gate A VIOLATION' || fail_msg "I2: refusal must name the Gate A violation [$out]"
|
|
# No partial write: the target must have received NO wake tool file.
|
|
[ ! -e "$TGT/tools/wake/beacon.sh" ] || fail_msg "I2: a refused install must not have written any wake file (partial write leaked)"
|
|
# And the positive control: with the REAL manifest, the same candidates install.
|
|
unset WAKE_INSTALL_MANIFEST
|
|
bash "$WI" install >/dev/null 2>&1 || fail_msg "I2: install must SUCCEED under the real framework-manifest"
|
|
[ -f "$TGT/tools/wake/beacon.sh" ] || fail_msg "I2: real-manifest install must write the wake tools"
|
|
) && ok
|
|
|
|
echo "== I3: blank-reset — reset idiom yields EXACTLY ONE OnUnitActiveUSec (negative control leaks two) =="
|
|
(
|
|
UD="$(fresh i3-units)"
|
|
export WAKE_SYSTEMD_USER_DIR="$UD"
|
|
UNIT="mosaic-heartbeat@a.timer"
|
|
# Base unit already carries a cadence (the legacy value being superseded).
|
|
mkdir -p "$UD"
|
|
cat >"$UD/$UNIT" <<'EOF'
|
|
[Timer]
|
|
OnUnitActiveSec=15min
|
|
[Install]
|
|
WantedBy=timers.target
|
|
EOF
|
|
# Blank-reset drop-in: empty reset then the new value.
|
|
bash "$WI" blank-reset "$UD/$UNIT.d/interval.conf" "30min" >/dev/null 2>&1 \
|
|
|| fail_msg "I3: blank-reset drop-in write failed"
|
|
out="$(bash "$WI" verify-single "$UNIT" 2>&1)"; rc=$?
|
|
[ "$rc" -eq 0 ] || fail_msg "I3: blank-reset must resolve exactly one OnUnitActiveUSec (rc=$rc) [$out]"
|
|
echo "$out" | grep -qi 'EXACTLY ONE' || fail_msg "I3: verify must confirm exactly-one [$out]"
|
|
# NEGATIVE CONTROL: a drop-in WITHOUT the reset line appends -> TWO values -> fail.
|
|
cat >"$UD/$UNIT.d/interval.conf" <<'EOF'
|
|
[Timer]
|
|
OnUnitActiveSec=30min
|
|
EOF
|
|
out2="$(bash "$WI" verify-single "$UNIT" 2>&1)"; rc2=$?
|
|
[ "$rc2" -ne 0 ] || fail_msg "I3: without the reset line two cadences must leak (verify should FAIL) [$out2]"
|
|
) && ok
|
|
|
|
echo "== I4: snapshot-guard — reap without a snapshot is REFUSED; allowed after snapshot =="
|
|
(
|
|
UD="$(fresh i4-units)"; SNAP="$(fresh i4-snap)"
|
|
export WAKE_SYSTEMD_USER_DIR="$UD"
|
|
export WAKE_SNAPSHOT_DIR="$SNAP"
|
|
UNIT="mosaic-heartbeat@b.timer"
|
|
printf '[Timer]\nOnUnitActiveSec=10min\n' >"$UD/$UNIT"
|
|
# (a) reap WITHOUT snapshot -> refuse, unit still present.
|
|
out="$(bash "$WI" reap-unit "$UNIT" 2>&1)"; rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "I4: reap without a snapshot must be REFUSED (rc=$rc)"
|
|
echo "$out" | grep -qi 'snapshot-guard' || fail_msg "I4: refusal must name the snapshot-guard [$out]"
|
|
[ -f "$UD/$UNIT" ] || fail_msg "I4: a refused reap must NOT delete the unit"
|
|
# (b) snapshot, then reap -> allowed, unit gone, snapshot retained.
|
|
bash "$WI" snapshot-unit "$UNIT" >/dev/null 2>&1 || fail_msg "I4: snapshot-unit failed"
|
|
bash "$WI" reap-unit "$UNIT" >/dev/null 2>&1 || fail_msg "I4: reap after snapshot must succeed"
|
|
[ ! -f "$UD/$UNIT" ] || fail_msg "I4: reap after snapshot must remove the unit"
|
|
[ -f "$SNAP/$UNIT" ] || fail_msg "I4: the snapshot must be retained after reap"
|
|
) && ok
|
|
|
|
echo "== I5: fail-closed install-validate — unconfigured HMAC/alarm FAIL LOUD; no secret inlined =="
|
|
(
|
|
H="$(fresh i5)"
|
|
CRED="$H/credentials.json"
|
|
export MOSAIC_CREDENTIALS_FILE="$CRED"
|
|
SECRET_KEY="s3cr3t-HMAC-DO-NOT-ECHO-xyz"
|
|
SECRET_ENDPOINT="https://monitor.invalid/alarm/DO-NOT-INLINE-abc123"
|
|
# --- HMAC key: absent store -> fail loud.
|
|
export WAKE_HMAC_KEY_NAME="primary"
|
|
out="$(bash "$WI" validate-hmac-key 2>&1)"; rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "I5: missing credential store must FAIL LOUD for the HMAC key (rc=$rc)"
|
|
echo "$out" | grep -qi 'UNSIGNED' || fail_msg "I5: HMAC failure must warn about unsigned wakes [$out]"
|
|
# Now provide the key by-name -> pass, and the value must NEVER be echoed.
|
|
jq -cn --arg k "$SECRET_KEY" '{wake:{hmac_keys:{"primary":$k}}}' >"$CRED"
|
|
out2="$(bash "$WI" validate-hmac-key 2>&1)"; rc2=$?
|
|
[ "$rc2" -eq 0 ] || fail_msg "I5: a by-name-resolvable HMAC key must pass (rc=$rc2) [$out2]"
|
|
echo "$out2" | grep -qF "$SECRET_KEY" && fail_msg "I5: validate must NEVER echo the HMAC key material"
|
|
# --- alarm target: unconfigured -> fail loud (silent no-alarm host).
|
|
unset WAKE_ALARM_SINK_CMD
|
|
out3="$(bash "$WI" validate-alarm-target 2>&1)"; rc3=$?
|
|
[ "$rc3" -ne 0 ] || fail_msg "I5: unconfigured alarm target must FAIL LOUD (rc=$rc3)"
|
|
echo "$out3" | grep -qi 'silent no-alarm host' || fail_msg "I5: alarm failure must name the silent-no-alarm hazard [$out3]"
|
|
# unreachable -> fail loud.
|
|
export WAKE_ALARM_SINK_CMD="false"
|
|
out4="$(bash "$WI" validate-alarm-target 2>&1)"; rc4=$?
|
|
[ "$rc4" -ne 0 ] || fail_msg "I5: unreachable alarm sink must FAIL LOUD (rc=$rc4)"
|
|
echo "$out4" | grep -qi 'UNREACHABLE' || fail_msg "I5: alarm failure must name the unreachable target [$out4]"
|
|
# reachable -> pass.
|
|
export WAKE_ALARM_SINK_CMD="cat >/dev/null"
|
|
bash "$WI" validate-alarm-target >/dev/null 2>&1 || fail_msg "I5: a configured+reachable alarm sink must pass"
|
|
# The installer file must inline NO secret/endpoint.
|
|
grep -qF "$SECRET_ENDPOINT" "$WI" && fail_msg "I5: wake-install.sh must NOT inline any alarm endpoint"
|
|
grep -qE 'hmac_keys[^A-Za-z_].*=[^=].*[A-Za-z0-9]{8,}' "$WI" && fail_msg "I5: wake-install.sh must NOT inline key material"
|
|
true
|
|
) && ok
|
|
|
|
echo "== I6: reset->verify->retire — overlap keeps legacy running; only §4-vector pass retires =="
|
|
(
|
|
UD="$(fresh i6-units)"; SNAP="$(fresh i6-snap)"
|
|
export WAKE_SYSTEMD_USER_DIR="$UD"
|
|
export WAKE_SNAPSHOT_DIR="$SNAP"
|
|
TIMER="mosaic-heartbeat@c.timer"
|
|
printf '[Timer]\nOnUnitActiveSec=15min\n' >"$UD/$TIMER"
|
|
# (a) overlap phase (NO --vector-passed): reset+verify, legacy LEFT RUNNING.
|
|
out="$(bash "$WI" reset-verify-retire c --interval 30min 2>&1)"; rc=$?
|
|
[ "$rc" -eq 0 ] || fail_msg "I6: overlap reset-verify must succeed (rc=$rc) [$out]"
|
|
[ -f "$UD/$TIMER" ] || fail_msg "I6: overlap phase must LEAVE the legacy timer running (retire withheld)"
|
|
[ -f "$SNAP/$TIMER" ] || fail_msg "I6: the legacy timer must be snapshotted before any retire"
|
|
echo "$out" | grep -qi 'LEFT RUNNING' || fail_msg "I6: overlap must announce retire is withheld [$out]"
|
|
# (b) §4-vector pass: retire LAST, snapshot-guarded.
|
|
out2="$(bash "$WI" reset-verify-retire c --interval 30min --vector-passed 2>&1)"; rc2=$?
|
|
[ "$rc2" -eq 0 ] || fail_msg "I6: vector-passed retire must succeed (rc=$rc2) [$out2]"
|
|
[ ! -f "$UD/$TIMER" ] || fail_msg "I6: on §4-vector pass the legacy timer must be RETIRED"
|
|
[ -f "$SNAP/$TIMER" ] || fail_msg "I6: the snapshot must survive the retire"
|
|
) && ok
|
|
|
|
# ── W6 monitor-integration hardening folded into beacon.sh (vi) ────────────────
|
|
|
|
echo "== I7: ingested_ts staleness — a far-future emit_ts still goes stale (receive-time governs) =="
|
|
(
|
|
H="$(fresh i7)"
|
|
export WAKE_BEACON_RECEIVED="$H/received.json"
|
|
export WAKE_ALARM_SINK_CMD="cat >$H/alarm.json"
|
|
now="$(date +%s)"
|
|
# Host lies with a far-future emit_ts; record stamps the monitor's ingested_ts.
|
|
jq -cn --argjson ts "$((now + 100000))" \
|
|
'{kind:"wake-beacon", beacon_seq:3, emit_ts:$ts, host_id:"h", independence:"off-host", degraded:false}' \
|
|
| bash "$BEACON" record
|
|
# Simulate the beacon having been received 100s ago (backdate ingested_ts only).
|
|
jq -c --argjson ing "$((now - 100))" '.ingested_ts = $ing' "$WAKE_BEACON_RECEIVED" >"$H/t" && mv "$H/t" "$WAKE_BEACON_RECEIVED"
|
|
out="$(bash "$BEACON" check --slo-seconds 5 2>&1)"; rc=$?
|
|
[ "$rc" -eq 1 ] || fail_msg "I7: far-future emit_ts must NOT defer staleness — receive-time governs (expected exit 1); got $rc [$out]"
|
|
) && ok
|
|
|
|
echo "== I8: beacon HMAC-verify at record — a spoofed (bad-sig) beacon is REJECTED =="
|
|
# #912: hard-require openssl in CI (Woodpecker sets CI=woodpecker) so the install
|
|
# beacon-sign leg is actually exercised; keep the skip for openssl-less local dev.
|
|
if ! command -v openssl >/dev/null 2>&1; then
|
|
if [ -n "${CI:-}" ]; then
|
|
echo " FAIL: I8 requires openssl in CI (#912) but it is not on PATH — the CI image must provide it" >&2
|
|
echo "x" >>"$FAILFILE"
|
|
else
|
|
echo "SKIP: openssl not available (local dev; CI hard-requires it)" >&2
|
|
fi
|
|
else
|
|
(
|
|
H="$(fresh i8)"
|
|
export WAKE_STATE_HOME="$H"
|
|
export MOSAIC_CREDENTIALS_FILE="$H/credentials.json"
|
|
export WAKE_BEACON_HMAC_KEY_NAME="beacon"
|
|
jq -cn --arg k "i8-hmac-key" '{wake:{hmac_keys:{"beacon":$k}}}' >"$MOSAIC_CREDENTIALS_FILE"
|
|
export WAKE_BEACON_INDEPENDENCE="off-host"
|
|
export WAKE_BEACON_SINK_CMD="cat >$H/shipped.json"
|
|
export WAKE_BEACON_RECEIVED="$H/received.json"
|
|
bash "$BEACON" emit >/dev/null 2>&1 || fail_msg "I8: signed emit must succeed"
|
|
bash "$BEACON" record <"$H/shipped.json" >/dev/null 2>&1 || fail_msg "I8: an authentic signed beacon must be accepted"
|
|
rm -f "$WAKE_BEACON_RECEIVED"
|
|
jq -c '.beacon_seq = 88888 | .host_id = "attacker"' "$H/shipped.json" >"$H/spoof.json"
|
|
out="$(bash "$BEACON" record <"$H/spoof.json" 2>&1)"; rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "I8: a spoofed beacon must be REJECTED at record (rc=$rc) [$out]"
|
|
[ ! -s "$WAKE_BEACON_RECEIVED" ] || fail_msg "I8: a rejected spoof must NOT advance the dead-man clock"
|
|
) && ok
|
|
fi
|
|
|
|
echo
|
|
if [ -s "$FAILFILE" ]; then
|
|
echo "wake install harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
|
|
exit 1
|
|
fi
|
|
echo "wake install harness: all invariants passed ($pass groups)"
|