jason.woltje
320f5bfb6f
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
133 lines
5.8 KiB
Bash
Executable File
133 lines
5.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-wake-fn-oracle.sh — RED-FIRST invariant harness for W5 (EPIC #892):
|
|
# the FN-oracle / synthetic-canary (fn-oracle.sh, A6).
|
|
#
|
|
# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if
|
|
# that invariant regresses:
|
|
# O1 healthy pipeline -> synthetic-canary FN-rate = 0 (PASS, exit 0) (§4 vector)
|
|
# O2 a DISABLED / dropping detector (perfect no-op rate) -> FN-DETECTED —
|
|
# the off-domain false-negative-blindspot killer: a detector that silently
|
|
# drops a KNOWN injected delta FAILS the oracle even at 0 wakes/day (§4/A8)
|
|
# O3 reached-CONSUMED-but-too-slow -> FN-DETECTED (the "within its per-class
|
|
# SLO" clause; a late delivery is still a false negative) (§4)
|
|
# O4 the verdict is OFF-DOMAIN: it is rendered from the terminal store cursor
|
|
# (consumed_seq), independent of any detector self-report (§4/A8)
|
|
# O5 no invented SLO: --slo-seconds is REQUIRED (fail-loud usage guard) (design law)
|
|
#
|
|
# Uses the oracle's OWN isolated state namespace + a pluggable drive command
|
|
# (WAKE_ORACLE_DETECTOR_CMD) so a disabled detector can be exercised. No live
|
|
# network, no operator queue touched.
|
|
#
|
|
# SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell
|
|
# and re-exports the per-test env, so environments are isolated by design (the
|
|
# same idiom as test-wake-detector.sh).
|
|
# shellcheck disable=SC2030,SC2031
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ORACLE="$SCRIPT_DIR/fn-oracle.sh"
|
|
DET="$SCRIPT_DIR/detector.sh"
|
|
|
|
command -v jq >/dev/null 2>&1 || {
|
|
echo "SKIP: jq not available" >&2
|
|
exit 0
|
|
}
|
|
|
|
TMP_ROOT="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP_ROOT"' EXIT
|
|
|
|
# Failures recorded to a FILE (subshell-safe — a var counter would silently
|
|
# swallow failures across the per-test subshells; mirrors the W2/W4 harnesses).
|
|
FAILFILE="$TMP_ROOT/failures"
|
|
: >"$FAILFILE"
|
|
pass=0
|
|
fail_msg() {
|
|
echo " FAIL: $*" >&2
|
|
echo "x" >>"$FAILFILE"
|
|
}
|
|
ok() { pass=$((pass + 1)); }
|
|
|
|
fresh_home() {
|
|
local d="$TMP_ROOT/$1"
|
|
rm -rf "$d"
|
|
mkdir -p "$d"
|
|
printf '%s' "$d"
|
|
}
|
|
|
|
echo "== O1: healthy pipeline -> synthetic-canary FN-rate = 0 =="
|
|
(
|
|
WAKE_ORACLE_HOME="$(fresh_home o1)"
|
|
export WAKE_ORACLE_HOME
|
|
unset WAKE_ORACLE_DETECTOR_CMD
|
|
out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || fail_msg "O1: a healthy pipeline must exit 0 (got $rc) [$out]"
|
|
echo "$out" | grep -q 'FN-RATE = 0/3' || fail_msg "O1: FN-rate must be 0/3 on a healthy pipeline [$out]"
|
|
echo "$out" | grep -q 'VERDICT = PASS' || fail_msg "O1: verdict must be PASS on a healthy pipeline [$out]"
|
|
) && ok
|
|
|
|
echo "== O2: DISABLED detector (perfect no-op) -> FN-DETECTED (blindspot killer) =="
|
|
(
|
|
WAKE_ORACLE_HOME="$(fresh_home o2)"
|
|
export WAKE_ORACLE_HOME
|
|
# A fully-disabled detector: it observes nothing, enqueues nothing — a
|
|
# "perfect" 0-wake rate that would look ideal on the wakes/day metric alone.
|
|
export WAKE_ORACLE_DETECTOR_CMD="true"
|
|
out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
|
|
rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "O2: a detector that drops a KNOWN delta MUST fail the oracle (non-zero exit), even at a perfect no-op rate"
|
|
echo "$out" | grep -q 'FN-RATE = 3/3' || fail_msg "O2: every dropped canary must count as a false-negative (FN-RATE 3/3) [$out]"
|
|
echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O2: verdict must be FN-DETECTED for a dropping detector [$out]"
|
|
echo "$out" | grep -qi 'never OBSERVED' || fail_msg "O2: the failure must name the dropped (never-observed) delta [$out]"
|
|
) && ok
|
|
|
|
echo "== O3: reached CONSUMED but too slow -> FN-DETECTED (within-SLO clause) =="
|
|
(
|
|
WAKE_ORACLE_HOME="$(fresh_home o3)"
|
|
export WAKE_ORACLE_HOME
|
|
# The REAL detector delivers the canary, but a delay pushes event->CONSUMED
|
|
# past a tight per-class SLO. A late delivery is still a false negative.
|
|
export WAKE_ORACLE_DETECTOR_CMD="sleep 2 && '$DET' poll-once"
|
|
out="$("$ORACLE" run --slo-seconds 1 --count 1 2>&1)"
|
|
rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "O3: a canary that reaches CONSUMED past its SLO must be a false-negative (non-zero exit)"
|
|
echo "$out" | grep -qi 'per-class SLO' || fail_msg "O3: the failure must attribute to the per-class SLO, not a drop [$out]"
|
|
# It must NOT be the 'never observed' branch: the delta WAS observed/delivered,
|
|
# just too slowly. This proves O3 exercises the SLO clause specifically.
|
|
if echo "$out" | grep -qi 'never OBSERVED'; then
|
|
fail_msg "O3: an SLO breach must NOT be misreported as a dropped delta [$out]"
|
|
fi
|
|
) && ok
|
|
|
|
echo "== O4: verdict is OFF-DOMAIN (from terminal consumed_seq, not detector self-report) =="
|
|
(
|
|
WAKE_ORACLE_HOME="$(fresh_home o4)"
|
|
export WAKE_ORACLE_HOME
|
|
# A drive that LIES 'success' (exit 0) but enqueues nothing. If the oracle
|
|
# trusted the drive's exit code it would pass; because it judges only the
|
|
# terminal store state, it correctly reports FN-DETECTED.
|
|
export WAKE_ORACLE_DETECTOR_CMD="exit 0"
|
|
out="$("$ORACLE" run --slo-seconds 120 --count 1 2>&1)"
|
|
rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "O4: a drive that exits 0 but delivers nothing must still be FN-DETECTED (verdict is off-domain)"
|
|
echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O4: off-domain verdict must not be fooled by a clean exit code [$out]"
|
|
) && ok
|
|
|
|
echo "== O5: no invented SLO -> --slo-seconds is REQUIRED (fail-loud) =="
|
|
(
|
|
WAKE_ORACLE_HOME="$(fresh_home o5)"
|
|
export WAKE_ORACLE_HOME
|
|
unset WAKE_ORACLE_DETECTOR_CMD WAKE_ORACLE_SLO_SECONDS
|
|
err="$("$ORACLE" run --count 1 2>&1)"
|
|
rc=$?
|
|
[ "$rc" -ne 0 ] || fail_msg "O5: run without an SLO must fail loud (no invented default)"
|
|
echo "$err" | grep -qi 'slo' || fail_msg "O5: the usage error must name the missing SLO [$err]"
|
|
) && ok
|
|
|
|
echo
|
|
if [ -s "$FAILFILE" ]; then
|
|
echo "wake fn-oracle harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
|
|
exit 1
|
|
fi
|
|
echo "wake fn-oracle harness: all invariants passed ($pass groups)"
|