mosaicstack/stack
25
0
Fork 1
Code Issues 344 Pull Requests 13 Actions Packages Projects Releases 15 Wiki Activity

fix(wake): three-valued grep verdicts — has_match/count_lines across all ten suites (closes #973) (#983)
ci/woodpecker/push/publish Pipeline was successful
Details
ci/woodpecker/push/ci Pipeline was successful
Details

Browse Source
This commit was merged in pull request #983.
This commit is contained in:
Mos
2026-07-31 08:39:04 +00:00
parent 089615f63b
commit 4fb44f6345
18 changed files with 4855 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/mosaic/framework/tools/wake/test-wake-fn-oracle.sh
+13 -10
View File
@@ -25,6 +25,9 @@
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# #973: three-valued grep assertion helpers (has_match/count_lines); init saves real stderr for abort loudness.
# shellcheck disable=SC1091
. "$SCRIPT_DIR/_wake-common.sh" && wake_assert_init
ORACLE="$SCRIPT_DIR/fn-oracle.sh"
DET="$SCRIPT_DIR/detector.sh"
@@ -62,8 +65,8 @@ echo "== O1: healthy pipeline -> synthetic-canary FN-rate = 0 =="
out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "O1: a healthy pipeline must exit 0 (got $rc) [$out]"
echo "$out" | grep -q 'FN-RATE = 0/3' || fail_msg "O1: FN-rate must be 0/3 on a healthy pipeline [$out]"
echo "$out" | grep -q 'VERDICT = PASS' || fail_msg "O1: verdict must be PASS on a healthy pipeline [$out]"
echo "$out" | has_match -q 'FN-RATE = 0/3' || fail_msg "O1: FN-rate must be 0/3 on a healthy pipeline [$out]"
echo "$out" | has_match -q 'VERDICT = PASS' || fail_msg "O1: verdict must be PASS on a healthy pipeline [$out]"
) && ok
echo "== O2: DISABLED detector (perfect no-op) -> FN-DETECTED (blindspot killer) =="
@@ -76,9 +79,9 @@ echo "== O2: DISABLED detector (perfect no-op) -> FN-DETECTED (blindspot killer)
out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O2: a detector that drops a KNOWN delta MUST fail the oracle (non-zero exit), even at a perfect no-op rate"
echo "$out" | grep -q 'FN-RATE = 3/3' || fail_msg "O2: every dropped canary must count as a false-negative (FN-RATE 3/3) [$out]"
echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O2: verdict must be FN-DETECTED for a dropping detector [$out]"
echo "$out" | grep -qi 'never OBSERVED' || fail_msg "O2: the failure must name the dropped (never-observed) delta [$out]"
echo "$out" | has_match -q 'FN-RATE = 3/3' || fail_msg "O2: every dropped canary must count as a false-negative (FN-RATE 3/3) [$out]"
echo "$out" | has_match -q 'VERDICT = FN-DETECTED' || fail_msg "O2: verdict must be FN-DETECTED for a dropping detector [$out]"
echo "$out" | has_match -qi 'never OBSERVED' || fail_msg "O2: the failure must name the dropped (never-observed) delta [$out]"
) && ok
echo "== O3: reached CONSUMED but too slow -> FN-DETECTED (within-SLO clause) =="
@@ -91,10 +94,10 @@ echo "== O3: reached CONSUMED but too slow -> FN-DETECTED (within-SLO clause) ==
out="$("$ORACLE" run --slo-seconds 1 --count 1 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O3: a canary that reaches CONSUMED past its SLO must be a false-negative (non-zero exit)"
echo "$out" | grep -qi 'per-class SLO' || fail_msg "O3: the failure must attribute to the per-class SLO, not a drop [$out]"
echo "$out" | has_match -qi 'per-class SLO' || fail_msg "O3: the failure must attribute to the per-class SLO, not a drop [$out]"
# It must NOT be the 'never observed' branch: the delta WAS observed/delivered,
# just too slowly. This proves O3 exercises the SLO clause specifically.
if echo "$out" | grep -qi 'never OBSERVED'; then
if echo "$out" | has_match -qi 'never OBSERVED'; then
fail_msg "O3: an SLO breach must NOT be misreported as a dropped delta [$out]"
fi
) && ok
@@ -110,7 +113,7 @@ echo "== O4: verdict is OFF-DOMAIN (from terminal consumed_seq, not detector sel
out="$("$ORACLE" run --slo-seconds 120 --count 1 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O4: a drive that exits 0 but delivers nothing must still be FN-DETECTED (verdict is off-domain)"
echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O4: off-domain verdict must not be fooled by a clean exit code [$out]"
echo "$out" | has_match -q 'VERDICT = FN-DETECTED' || fail_msg "O4: off-domain verdict must not be fooled by a clean exit code [$out]"
) && ok
echo "== O5: no invented SLO -> --slo-seconds is REQUIRED (fail-loud) =="
@@ -121,12 +124,12 @@ echo "== O5: no invented SLO -> --slo-seconds is REQUIRED (fail-loud) =="
err="$("$ORACLE" run --count 1 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O5: run without an SLO must fail loud (no invented default)"
echo "$err" | grep -qi 'slo' || fail_msg "O5: the usage error must name the missing SLO [$err]"
echo "$err" | has_match -qi 'slo' || fail_msg "O5: the usage error must name the missing SLO [$err]"
) && ok
echo
if [ -s "$FAILFILE" ]; then
echo "wake fn-oracle harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
echo "wake fn-oracle harness: FAILED ($(count_lines . "$FAILFILE") assertion(s))" >&2
exit 1
fi
echo "wake fn-oracle harness: all invariants passed ($pass groups)"