#!/usr/bin/env bash # test-wake-fn-oracle.sh — RED-FIRST invariant harness for W5 (EPIC #892): # the FN-oracle / synthetic-canary (fn-oracle.sh, A6). # # Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if # that invariant regresses: # O1 healthy pipeline -> synthetic-canary FN-rate = 0 (PASS, exit 0) (§4 vector) # O2 a DISABLED / dropping detector (perfect no-op rate) -> FN-DETECTED — # the off-domain false-negative-blindspot killer: a detector that silently # drops a KNOWN injected delta FAILS the oracle even at 0 wakes/day (§4/A8) # O3 reached-CONSUMED-but-too-slow -> FN-DETECTED (the "within its per-class # SLO" clause; a late delivery is still a false negative) (§4) # O4 the verdict is OFF-DOMAIN: it is rendered from the terminal store cursor # (consumed_seq), independent of any detector self-report (§4/A8) # O5 no invented SLO: --slo-seconds is REQUIRED (fail-loud usage guard) (design law) # # Uses the oracle's OWN isolated state namespace + a pluggable drive command # (WAKE_ORACLE_DETECTOR_CMD) so a disabled detector can be exercised. No live # network, no operator queue touched. # # SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell # and re-exports the per-test env, so environments are isolated by design (the # same idiom as test-wake-detector.sh). # shellcheck disable=SC2030,SC2031 set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ORACLE="$SCRIPT_DIR/fn-oracle.sh" DET="$SCRIPT_DIR/detector.sh" command -v jq >/dev/null 2>&1 || { echo "SKIP: jq not available" >&2 exit 0 } TMP_ROOT="$(mktemp -d)" trap 'rm -rf "$TMP_ROOT"' EXIT # Failures recorded to a FILE (subshell-safe — a var counter would silently # swallow failures across the per-test subshells; mirrors the W2/W4 harnesses). FAILFILE="$TMP_ROOT/failures" : >"$FAILFILE" pass=0 fail_msg() { echo " FAIL: $*" >&2 echo "x" >>"$FAILFILE" } ok() { pass=$((pass + 1)); } fresh_home() { local d="$TMP_ROOT/$1" rm -rf "$d" mkdir -p "$d" printf '%s' "$d" } echo "== O1: healthy pipeline -> synthetic-canary FN-rate = 0 ==" ( WAKE_ORACLE_HOME="$(fresh_home o1)" export WAKE_ORACLE_HOME unset WAKE_ORACLE_DETECTOR_CMD out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)" rc=$? [ "$rc" -eq 0 ] || fail_msg "O1: a healthy pipeline must exit 0 (got $rc) [$out]" echo "$out" | grep -q 'FN-RATE = 0/3' || fail_msg "O1: FN-rate must be 0/3 on a healthy pipeline [$out]" echo "$out" | grep -q 'VERDICT = PASS' || fail_msg "O1: verdict must be PASS on a healthy pipeline [$out]" ) && ok echo "== O2: DISABLED detector (perfect no-op) -> FN-DETECTED (blindspot killer) ==" ( WAKE_ORACLE_HOME="$(fresh_home o2)" export WAKE_ORACLE_HOME # A fully-disabled detector: it observes nothing, enqueues nothing — a # "perfect" 0-wake rate that would look ideal on the wakes/day metric alone. export WAKE_ORACLE_DETECTOR_CMD="true" out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)" rc=$? [ "$rc" -ne 0 ] || fail_msg "O2: a detector that drops a KNOWN delta MUST fail the oracle (non-zero exit), even at a perfect no-op rate" echo "$out" | grep -q 'FN-RATE = 3/3' || fail_msg "O2: every dropped canary must count as a false-negative (FN-RATE 3/3) [$out]" echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O2: verdict must be FN-DETECTED for a dropping detector [$out]" echo "$out" | grep -qi 'never OBSERVED' || fail_msg "O2: the failure must name the dropped (never-observed) delta [$out]" ) && ok echo "== O3: reached CONSUMED but too slow -> FN-DETECTED (within-SLO clause) ==" ( WAKE_ORACLE_HOME="$(fresh_home o3)" export WAKE_ORACLE_HOME # The REAL detector delivers the canary, but a delay pushes event->CONSUMED # past a tight per-class SLO. A late delivery is still a false negative. export WAKE_ORACLE_DETECTOR_CMD="sleep 2 && '$DET' poll-once" out="$("$ORACLE" run --slo-seconds 1 --count 1 2>&1)" rc=$? [ "$rc" -ne 0 ] || fail_msg "O3: a canary that reaches CONSUMED past its SLO must be a false-negative (non-zero exit)" echo "$out" | grep -qi 'per-class SLO' || fail_msg "O3: the failure must attribute to the per-class SLO, not a drop [$out]" # It must NOT be the 'never observed' branch: the delta WAS observed/delivered, # just too slowly. This proves O3 exercises the SLO clause specifically. if echo "$out" | grep -qi 'never OBSERVED'; then fail_msg "O3: an SLO breach must NOT be misreported as a dropped delta [$out]" fi ) && ok echo "== O4: verdict is OFF-DOMAIN (from terminal consumed_seq, not detector self-report) ==" ( WAKE_ORACLE_HOME="$(fresh_home o4)" export WAKE_ORACLE_HOME # A drive that LIES 'success' (exit 0) but enqueues nothing. If the oracle # trusted the drive's exit code it would pass; because it judges only the # terminal store state, it correctly reports FN-DETECTED. export WAKE_ORACLE_DETECTOR_CMD="exit 0" out="$("$ORACLE" run --slo-seconds 120 --count 1 2>&1)" rc=$? [ "$rc" -ne 0 ] || fail_msg "O4: a drive that exits 0 but delivers nothing must still be FN-DETECTED (verdict is off-domain)" echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O4: off-domain verdict must not be fooled by a clean exit code [$out]" ) && ok echo "== O5: no invented SLO -> --slo-seconds is REQUIRED (fail-loud) ==" ( WAKE_ORACLE_HOME="$(fresh_home o5)" export WAKE_ORACLE_HOME unset WAKE_ORACLE_DETECTOR_CMD WAKE_ORACLE_SLO_SECONDS err="$("$ORACLE" run --count 1 2>&1)" rc=$? [ "$rc" -ne 0 ] || fail_msg "O5: run without an SLO must fail loud (no invented default)" echo "$err" | grep -qi 'slo' || fail_msg "O5: the usage error must name the missing SLO [$err]" ) && ok echo if [ -s "$FAILFILE" ]; then echo "wake fn-oracle harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 exit 1 fi echo "wake fn-oracle harness: all invariants passed ($pass groups)"