stack/packages/mosaic/framework/tools/wake/test-wake-fn-oracle.sh

#!/usr/bin/env bash
# test-wake-fn-oracle.sh — RED-FIRST invariant harness for W5 (EPIC #892):
# the FN-oracle / synthetic-canary (fn-oracle.sh, A6).
#
# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if
# that invariant regresses:
#   O1 healthy pipeline -> synthetic-canary FN-rate = 0 (PASS, exit 0)   (§4 vector)
#   O2 a DISABLED / dropping detector (perfect no-op rate) -> FN-DETECTED —
#      the off-domain false-negative-blindspot killer: a detector that silently
#      drops a KNOWN injected delta FAILS the oracle even at 0 wakes/day  (§4/A8)
#   O3 reached-CONSUMED-but-too-slow -> FN-DETECTED (the "within its per-class
#      SLO" clause; a late delivery is still a false negative)            (§4)
#   O4 the verdict is OFF-DOMAIN: it is rendered from the terminal store cursor
#      (consumed_seq), independent of any detector self-report              (§4/A8)
#   O5 no invented SLO: --slo-seconds is REQUIRED (fail-loud usage guard)  (design law)
#
# Uses the oracle's OWN isolated state namespace + a pluggable drive command
# (WAKE_ORACLE_DETECTOR_CMD) so a disabled detector can be exercised. No live
# network, no operator queue touched.
#
# SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell
# and re-exports the per-test env, so environments are isolated by design (the
# same idiom as test-wake-detector.sh).
# shellcheck disable=SC2030,SC2031
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ORACLE="$SCRIPT_DIR/fn-oracle.sh"
DET="$SCRIPT_DIR/detector.sh"

command -v jq >/dev/null 2>&1 || {
  echo "SKIP: jq not available" >&2
  exit 0
}

TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMP_ROOT"' EXIT

# Failures recorded to a FILE (subshell-safe — a var counter would silently
# swallow failures across the per-test subshells; mirrors the W2/W4 harnesses).
FAILFILE="$TMP_ROOT/failures"
: >"$FAILFILE"
pass=0
fail_msg() {
  echo "  FAIL: $*" >&2
  echo "x" >>"$FAILFILE"
}
ok() { pass=$((pass + 1)); }

fresh_home() {
  local d="$TMP_ROOT/$1"
  rm -rf "$d"
  mkdir -p "$d"
  printf '%s' "$d"
}

echo "== O1: healthy pipeline -> synthetic-canary FN-rate = 0 =="
(
  WAKE_ORACLE_HOME="$(fresh_home o1)"
  export WAKE_ORACLE_HOME
  unset WAKE_ORACLE_DETECTOR_CMD
  out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
  rc=$?
  [ "$rc" -eq 0 ] || fail_msg "O1: a healthy pipeline must exit 0 (got $rc) [$out]"
  echo "$out" | grep -q 'FN-RATE = 0/3' || fail_msg "O1: FN-rate must be 0/3 on a healthy pipeline [$out]"
  echo "$out" | grep -q 'VERDICT = PASS' || fail_msg "O1: verdict must be PASS on a healthy pipeline [$out]"
) && ok

echo "== O2: DISABLED detector (perfect no-op) -> FN-DETECTED (blindspot killer) =="
(
  WAKE_ORACLE_HOME="$(fresh_home o2)"
  export WAKE_ORACLE_HOME
  # A fully-disabled detector: it observes nothing, enqueues nothing — a
  # "perfect" 0-wake rate that would look ideal on the wakes/day metric alone.
  export WAKE_ORACLE_DETECTOR_CMD="true"
  out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || fail_msg "O2: a detector that drops a KNOWN delta MUST fail the oracle (non-zero exit), even at a perfect no-op rate"
  echo "$out" | grep -q 'FN-RATE = 3/3' || fail_msg "O2: every dropped canary must count as a false-negative (FN-RATE 3/3) [$out]"
  echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O2: verdict must be FN-DETECTED for a dropping detector [$out]"
  echo "$out" | grep -qi 'never OBSERVED' || fail_msg "O2: the failure must name the dropped (never-observed) delta [$out]"
) && ok

echo "== O3: reached CONSUMED but too slow -> FN-DETECTED (within-SLO clause) =="
(
  WAKE_ORACLE_HOME="$(fresh_home o3)"
  export WAKE_ORACLE_HOME
  # The REAL detector delivers the canary, but a delay pushes event->CONSUMED
  # past a tight per-class SLO. A late delivery is still a false negative.
  export WAKE_ORACLE_DETECTOR_CMD="sleep 2 && '$DET' poll-once"
  out="$("$ORACLE" run --slo-seconds 1 --count 1 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || fail_msg "O3: a canary that reaches CONSUMED past its SLO must be a false-negative (non-zero exit)"
  echo "$out" | grep -qi 'per-class SLO' || fail_msg "O3: the failure must attribute to the per-class SLO, not a drop [$out]"
  # It must NOT be the 'never observed' branch: the delta WAS observed/delivered,
  # just too slowly. This proves O3 exercises the SLO clause specifically.
  if echo "$out" | grep -qi 'never OBSERVED'; then
    fail_msg "O3: an SLO breach must NOT be misreported as a dropped delta [$out]"
  fi
) && ok

echo "== O4: verdict is OFF-DOMAIN (from terminal consumed_seq, not detector self-report) =="
(
  WAKE_ORACLE_HOME="$(fresh_home o4)"
  export WAKE_ORACLE_HOME
  # A drive that LIES 'success' (exit 0) but enqueues nothing. If the oracle
  # trusted the drive's exit code it would pass; because it judges only the
  # terminal store state, it correctly reports FN-DETECTED.
  export WAKE_ORACLE_DETECTOR_CMD="exit 0"
  out="$("$ORACLE" run --slo-seconds 120 --count 1 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || fail_msg "O4: a drive that exits 0 but delivers nothing must still be FN-DETECTED (verdict is off-domain)"
  echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O4: off-domain verdict must not be fooled by a clean exit code [$out]"
) && ok

echo "== O5: no invented SLO -> --slo-seconds is REQUIRED (fail-loud) =="
(
  WAKE_ORACLE_HOME="$(fresh_home o5)"
  export WAKE_ORACLE_HOME
  unset WAKE_ORACLE_DETECTOR_CMD WAKE_ORACLE_SLO_SECONDS
  err="$("$ORACLE" run --count 1 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || fail_msg "O5: run without an SLO must fail loud (no invented default)"
  echo "$err" | grep -qi 'slo' || fail_msg "O5: the usage error must name the missing SLO [$err]"
) && ok

echo
if [ -s "$FAILFILE" ]; then
  echo "wake fn-oracle harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
  exit 1
fi
echo "wake fn-oracle harness: all invariants passed ($pass groups)"