Files
stack/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh
jason.woltje 347c1d57c1
All checks were successful
ci/woodpecker/push/publish Pipeline was successful
ci/woodpecker/push/ci Pipeline was successful
fix(wake): #924 route dead-letter quarantine alarm via WAKE_ALARM_SINK_CMD with per-observed_seq dedup (G2a) (#929)
Co-authored-by: jason.woltje <jason@diversecanvas.com>
Co-committed-by: jason.woltje <jason@diversecanvas.com>
2026-07-26 11:45:09 +00:00

310 lines
17 KiB
Bash
Executable File

#!/usr/bin/env bash
# test-wake-digest-quarantine.sh — RED-FIRST invariant harness for #920 (EPIC
# #892): per-entry QUARANTINE of a render-refused digest entry (no head-of-line
# blocking) + reconciler ENUMERATIONS render ORIENTATION-tier.
#
# Each test asserts ONE invariant and goes RED against the pre-#920 digest.sh:
# Q1 (a) QUARANTINE + REST-DELIVERS: a malformed `actionable` carrying the live
# pilot's EXACT locator shape {kind,id,path,observed_hash} (no hard
# locator, no reconciled marker) is DEAD-LETTERED + alarmed AND EXCLUDED,
# while a clean valid sibling in the SAME drain still renders (exit 0).
# RED baseline: the old whole-digest exit-4 delivered NOTHING (the live
# head-of-line-blocking wedge). (#920 FIX1)
# Q2 (b) ENUM-AS-ORIENTATION: a reconciler enumeration (locators.reconciled==
# true) renders as an ORIENTATION-tier pointer and does NOT exit-4 / is
# NOT quarantined. RED baseline: reconciled `actionable` + soft locators
# {kind,id,path,observed_hash} -> exit-4. (#920 FIX2)
# Q3 (c) TWO DISTINCT ENUMERATIONS BOTH SURVIVE: two distinct enumerations both
# render as SEPARATE orientation pointers (neither coalesced away) — the
# anti-collapse proof pinning the amended ruling (the REJECTED class=digest
# would have collapsed them to one; store class stays non-coalescing). (#920 FIX2)
# Q4 PRESERVED ACTIONABLE FAIL-LOUD: a genuine malformed ACTIONABLE claim
# (no reconciled marker, no hard locator) is STILL loudly surfaced
# (dead-letter + alarm) and NEVER delivered as if valid — fail-loud is
# preserved, now per-entry. (§2.1)
# Q5 CLAIM-PRECEDENCE GATED: a non-actionable-class entry that carries a
# `claim` (a consequential fact) with no hard locator is STILL gated
# (quarantined), and the reconciled exemption does not leak to it. (§2.1)
#
# #924 (G2a fix — the #920 alarm was stderr/journal-LOCAL only, a permanent
# silent-miss hazard since a dead-lettered entry is store-accounted and the
# reconciler never re-flags it): the SAME per-entry alarm now ALSO routes
# off-host via WAKE_ALARM_SINK_CMD (beacon.sh's W6/#910 pluggable adapter,
# REUSED verbatim), deduped by observed_seq (the entry's durable identity).
# Q6 (a) ONE off-host alarm + stderr diagnostic STILL fires: a dead-lettered
# entry routes EXACTLY ONE alarm to a captured WAKE_ALARM_SINK_CMD
# (payload names observed_seq), AND the #920 stderr diagnostic still
# fires (local + off-host, never either/or).
# Q7 (b) RE-DRAIN DEDUP: re-draining the SAME still-dead-lettered entry N
# times routes ZERO additional off-host alarms (durable dedup by
# observed_seq survives across separate digest.sh invocations, i.e.
# across drains/restarts, since each invocation is a fresh process).
# Q8 (c) NEW ENTRY OWN ALARM: a NEW distinct dead-lettered entry (a new
# observed_seq) routes its OWN one alarm — dedup is per-entry, not a
# global "alarm already fired at all" latch.
# Q9 (d) FAIL-CLOSED: WAKE_ALARM_SINK_CMD unconfigured OR unreachable (exit
# non-zero) is a LOUD stderr diagnostic (mirrors beacon.sh's
# fail-closed wording) — never a silent no-alarm host. Per-entry, not
# whole-drain: render still exits 0 (#920's no-head-of-line-block
# property is preserved even when the off-host leg itself fails).
#
# RED-FIRST: Q6-Q9 all go RED against the pre-#924 (stderr-only) digest.sh —
# it never references WAKE_ALARM_SINK_CMD at all, so no payload is EVER routed
# (Q6/Q7/Q8 all see 0 captured alarms) and no "FAIL LOUD ... alarm sink"
# diagnostic exists to fire (Q9).
#
# Hermetic: feeds controlled JSONL via `digest.sh render --from-file` — NO store,
# NO network, NO openssl (so it runs identically under the CI openssl-mask).
#
# Each test runs in its own (..) subshell for env isolation; the per-subshell
# WAKE_STATE_HOME export is intentional (mirrors test-wake-reconcile.sh).
# shellcheck disable=SC2030,SC2031
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DIGEST="$SCRIPT_DIR/digest.sh"
command -v jq >/dev/null 2>&1 || {
echo "SKIP: jq not available" >&2
exit 0
}
TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMP_ROOT"' EXIT
FAILFILE="$TMP_ROOT/failures"
: >"$FAILFILE"
pass=0
fail_msg() {
echo " FAIL: $*" >&2
echo "x" >>"$FAILFILE"
}
ok() { pass=$((pass + 1)); }
# A 40-hex sha = a valid §2.1 hard locator.
SHA40="abcdef0123456789abcdef0123456789abcdef01"
# capture-alarm (#924): a REACHABLE off-host alarm route that APPENDS the
# routed payload (one JSON line per invocation) to $ALARM_OUT, so a test can
# count exactly how many off-host alarms fired across one or more digest.sh
# invocations. Mirrors test-wake-beacon.sh's capture-alarm idiom.
CAPTURE_ALARM="$TMP_ROOT/capture-alarm.sh"
cat >"$CAPTURE_ALARM" <<'EOF'
#!/usr/bin/env bash
cat >>"$ALARM_OUT"
EOF
chmod +x "$CAPTURE_ALARM"
# fresh_home NAME — a fresh WAKE_STATE_HOME dir, echoed.
fresh_home() {
local d="$TMP_ROOT/$1"
rm -rf "$d"
mkdir -p "$d"
printf '%s' "$d"
}
# dlq HOME — the dead-letter path for the default agent under HOME.
dlq() { printf '%s/default/dead-letter.jsonl' "$1"; }
echo "== Q1 (a): malformed {kind,id,path,observed_hash} QUARANTINES; clean sibling STILL delivers =="
(
home="$(fresh_home q1)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q1.jsonl"
# The live pilot's EXACT malformed shape (no reconciled marker) + a clean sibling.
{
printf '%s\n' '{"observed_seq":1,"class":"actionable","locators":{"kind":"repo","id":"MALFORMED-Q","path":"docs/x.md","observed_hash":"deadbeef"},"emit_ts":1}'
printf '{"observed_seq":2,"class":"actionable","locators":{"sha":"%s","file":"src/a.ts"},"emit_ts":1}\n' "$SHA40"
} >"$f"
err="$TMP_ROOT/q1.err"
out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "Q1: render must EXIT 0 (per-entry quarantine, not whole-digest exit-4), got rc=$rc"
printf '%s' "$out" | grep -q "$SHA40" || fail_msg "Q1: the clean sibling (sha $SHA40) must STILL be delivered in the same drain [head-of-line block]"
printf '%s' "$out" | grep -q 'MALFORMED-Q' && fail_msg "Q1: the quarantined entry must be EXCLUDED from the rendered digest"
[ -f "$(dlq "$home")" ] || fail_msg "Q1: a durable dead-letter file must be written"
grep -q 'MALFORMED-Q' "$(dlq "$home")" 2>/dev/null || fail_msg "Q1: the malformed entry must be DEAD-LETTERED (accounted-for, not silently dropped)"
grep -qi 'QUARANTINE' "$err" || fail_msg "Q1: a LOUD per-entry alarm must fire on stderr"
grep -q 'observed_seq=1' "$err" || fail_msg "Q1: the alarm must identify the offending entry (observed_seq=1)"
) && ok
echo "== Q2 (b): reconciler enumeration (reconciled:true) renders ORIENTATION-tier, NO exit-4 =="
(
home="$(fresh_home q2)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q2.jsonl"
# A reconciler enumeration: store class actionable (unchanged) + reconciled marker.
printf '%s\n' '{"observed_seq":5,"class":"actionable","locators":{"kind":"repo","id":"ENUM-B","path":"BOARD.md","observed_hash":"cafe1234","reconciled":true},"emit_ts":1}' >"$f"
err="$TMP_ROOT/q2.err"
out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "Q2: a reconciler enumeration must NOT exit-4 (it is ORIENTATION-tier), got rc=$rc"
printf '%s' "$out" | grep -q 'id=ENUM-B' || fail_msg "Q2: the enumeration must render as an ORIENTATION pointer (id=ENUM-B via _locator_line)"
printf '%s' "$out" | grep -q 'CLAIM@seq' && fail_msg "Q2: an enumeration must NOT render as an ACTIONABLE CLAIM@seq"
[ -s "$(dlq "$home")" ] && fail_msg "Q2: an ORIENTATION-tier enumeration must NOT be quarantined/dead-lettered"
true
) && ok
echo "== Q3 (c): TWO DISTINCT enumerations BOTH survive as SEPARATE orientation pointers (anti-collapse) =="
(
home="$(fresh_home q3)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q3.jsonl"
{
printf '%s\n' '{"observed_seq":6,"class":"actionable","locators":{"kind":"repo","id":"ENUM-C1","path":"a.md","observed_hash":"1111","reconciled":true},"emit_ts":1}'
printf '%s\n' '{"observed_seq":7,"class":"actionable","locators":{"kind":"repo","id":"ENUM-C2","path":"b.md","observed_hash":"2222","reconciled":true},"emit_ts":1}'
} >"$f"
err="$TMP_ROOT/q3.err"
out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "Q3: two enumerations must render (exit 0), got rc=$rc"
n="$(printf '%s\n' "$out" | grep -c 'id=ENUM-C[12]' || true)"
[ "$n" = "2" ] || fail_msg "Q3: BOTH distinct enumerations must survive as SEPARATE orientation pointers (expected 2, got $n) — neither coalesced away"
printf '%s' "$out" | grep -q 'id=ENUM-C1' || fail_msg "Q3: enumeration ENUM-C1 must be present"
printf '%s' "$out" | grep -q 'id=ENUM-C2' || fail_msg "Q3: enumeration ENUM-C2 must be present"
[ -s "$(dlq "$home")" ] && fail_msg "Q3: enumerations must NOT be quarantined"
true
) && ok
echo "== Q4: PRESERVED — a genuine malformed ACTIONABLE claim is STILL loud (dead-letter+alarm), never delivered as valid =="
(
home="$(fresh_home q4)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q4.jsonl"
printf '%s\n' '{"observed_seq":9,"class":"actionable","locators":{"kind":"board_file","id":"CLAIM-KEEP","observed_hash":"beef"},"emit_ts":1}' >"$f"
err="$TMP_ROOT/q4.err"
out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "Q4: render must exit 0 (per-entry quarantine), got rc=$rc"
printf '%s' "$out" | grep -q 'CLAIM-KEEP' && fail_msg "Q4: a malformed actionable must NOT be delivered as if valid"
printf '%s' "$out" | grep -q '(none) — no consequential claims pending' || fail_msg "Q4: with the only claim quarantined, the ACTIONABLE section must show (none)"
grep -q 'CLAIM-KEEP' "$(dlq "$home")" 2>/dev/null || fail_msg "Q4: the malformed actionable must be DEAD-LETTERED (loudly surfaced, not silent)"
grep -qi 'QUARANTINE' "$err" || fail_msg "Q4: the malformed actionable must raise a LOUD alarm"
) && ok
echo "== Q5: claim-precedence gated — a non-actionable-class entry carrying a claim (no hard locator) is STILL quarantined =="
(
home="$(fresh_home q5)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q5.jsonl"
# class=digest but locators carry a consequential `claim` -> actionable-tier by
# precedence; no hard locator + no reconciled marker -> must be quarantined.
printf '%s\n' '{"observed_seq":11,"class":"digest","locators":{"claim":"CI is green","kind":"repo","id":"CLAIMY"},"emit_ts":1}' >"$f"
err="$TMP_ROOT/q5.err"
out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "Q5: render must exit 0, got rc=$rc"
grep -q 'CLAIMY' "$(dlq "$home")" 2>/dev/null || fail_msg "Q5: a claim-carrying entry with no hard locator must be quarantined (claim precedence, §2.1)"
printf '%s' "$out" | grep -q 'CI is green' && fail_msg "Q5: the un-verifiable claim must NOT be delivered"
true
) && ok
echo "== Q6 (a): dead-lettered entry routes EXACTLY ONE off-host alarm (payload names observed_seq) + stderr diagnostic STILL fires =="
(
home="$(fresh_home q6)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q6.jsonl"
printf '%s\n' '{"observed_seq":21,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q6","path":"x.md","observed_hash":"aaaa"},"emit_ts":1}' >"$f"
ALARM_OUT="$TMP_ROOT/q6.alarm.jsonl"
export ALARM_OUT
: >"$ALARM_OUT"
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
err="$TMP_ROOT/q6.err"
out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "Q6: render must still EXIT 0 (per-entry quarantine, not whole-drain wedge), got rc=$rc"
n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)"
[ "$n" = "1" ] || fail_msg "Q6: EXACTLY ONE off-host alarm must route for the dead-lettered entry (got $n) [$(cat "$ALARM_OUT" 2>/dev/null)]"
grep -q '"observed_seq":21' "$ALARM_OUT" 2>/dev/null || fail_msg "Q6: the routed alarm payload must name the entry's observed_seq (21)"
grep -qi 'QUARANTINE' "$err" || fail_msg "Q6: the existing #920 stderr diagnostic must STILL fire (local + off-host, not either/or)"
printf '%s' "$out" | grep -q 'DLQ-Q6' && fail_msg "Q6: the quarantined entry must still be EXCLUDED from the rendered digest"
true
) && ok
echo "== Q7 (b): re-draining the SAME still-dead-lettered entry N times routes ZERO additional off-host alarms (dedup by observed_seq) =="
(
home="$(fresh_home q7)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q7.jsonl"
printf '%s\n' '{"observed_seq":22,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q7","path":"y.md","observed_hash":"bbbb"},"emit_ts":1}' >"$f"
ALARM_OUT="$TMP_ROOT/q7.alarm.jsonl"
export ALARM_OUT
: >"$ALARM_OUT"
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
for i in 1 2 3 4; do
"$DIGEST" render --from-file "$f" --agent default >/dev/null 2>"$TMP_ROOT/q7.err.$i"
done
n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)"
[ "$n" = "1" ] || fail_msg "Q7: re-draining the SAME dead-lettered entry 4x must route ONLY ONE off-host alarm total (durable dedup by observed_seq); got $n [$(cat "$ALARM_OUT" 2>/dev/null)]"
grep -qi 'QUARANTINE' "$TMP_ROOT/q7.err.4" || fail_msg "Q7: the #920 stderr diagnostic must STILL fire on every re-drain (only the off-host route is deduped)"
) && ok
echo "== Q8 (c): a NEW distinct dead-lettered entry routes its OWN one alarm (dedup is per-entry, not a global latch) =="
(
home="$(fresh_home q8)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f1="$TMP_ROOT/q8a.jsonl"
f2="$TMP_ROOT/q8b.jsonl"
printf '%s\n' '{"observed_seq":31,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q8A","path":"a.md","observed_hash":"c1"},"emit_ts":1}' >"$f1"
printf '%s\n' '{"observed_seq":32,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q8B","path":"b.md","observed_hash":"c2"},"emit_ts":1}' >"$f2"
ALARM_OUT="$TMP_ROOT/q8.alarm.jsonl"
export ALARM_OUT
: >"$ALARM_OUT"
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
"$DIGEST" render --from-file "$f1" --agent default >/dev/null 2>/dev/null
"$DIGEST" render --from-file "$f1" --agent default >/dev/null 2>/dev/null # re-drain seq 31 -> must NOT re-alarm
"$DIGEST" render --from-file "$f2" --agent default >/dev/null 2>/dev/null # NEW distinct seq 32 -> its own alarm
n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)"
[ "$n" = "2" ] || fail_msg "Q8: two DISTINCT dead-lettered entries must together route exactly 2 off-host alarms total (got $n) [$(cat "$ALARM_OUT" 2>/dev/null)]"
grep -q '"observed_seq":31' "$ALARM_OUT" 2>/dev/null || fail_msg "Q8: seq 31's alarm must be present"
grep -q '"observed_seq":32' "$ALARM_OUT" 2>/dev/null || fail_msg "Q8: seq 32's (the new distinct entry's) OWN alarm must be present"
) && ok
echo "== Q9 (d): WAKE_ALARM_SINK_CMD unconfigured OR unreachable -> FAIL LOUD (never silent no-alarm); per-entry, render still exits 0 =="
(
home="$(fresh_home q9a)"
export WAKE_STATE_HOME="$home"
unset WAKE_AGENT
f="$TMP_ROOT/q9.jsonl"
printf '%s\n' '{"observed_seq":41,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q9","path":"z.md","observed_hash":"dddd"},"emit_ts":1}' >"$f"
# (a) UNCONFIGURED alarm sink.
unset WAKE_ALARM_SINK_CMD
err_a="$TMP_ROOT/q9a.err"
out_a="$("$DIGEST" render --from-file "$f" --agent default 2>"$err_a")"
rc_a=$?
[ "$rc_a" -eq 0 ] || fail_msg "Q9a: per-entry quarantine must still exit 0 even when the off-host alarm sink is unconfigured (no whole-drain wedge), got rc=$rc_a"
grep -qi 'FAIL LOUD' "$err_a" || fail_msg "Q9a: an unconfigured off-host alarm target must FAIL LOUD on stderr [$(cat "$err_a")]"
grep -Eqi 'silent no-alarm|silent-miss|PERMANENTLY miss|permanent silent miss' "$err_a" || fail_msg "Q9a: the diagnostic must name the silent-miss hazard (G2a), mirroring beacon.sh's fail-closed wording [$(cat "$err_a")]"
printf '%s' "$out_a" | grep -q 'DLQ-Q9' && fail_msg "Q9a: the quarantined entry must still be EXCLUDED from the rendered digest"
true
) && ok
(
home2="$(fresh_home q9b)"
export WAKE_STATE_HOME="$home2"
unset WAKE_AGENT
f="$TMP_ROOT/q9.jsonl"
export WAKE_ALARM_SINK_CMD="false"
err_b="$TMP_ROOT/q9b.err"
out_b="$("$DIGEST" render --from-file "$f" --agent default 2>"$err_b")"
rc_b=$?
[ "$rc_b" -eq 0 ] || fail_msg "Q9b: per-entry quarantine must still exit 0 even when the off-host alarm sink is unreachable, got rc=$rc_b"
grep -qi 'FAIL LOUD' "$err_b" || fail_msg "Q9b: an unreachable off-host alarm target must FAIL LOUD on stderr [$(cat "$err_b")]"
grep -qi 'UNREACHABLE' "$err_b" || fail_msg "Q9b: the diagnostic must name the unreachable target [$(cat "$err_b")]"
printf '%s' "$out_b" | grep -q 'DLQ-Q9' && fail_msg "Q9b: the quarantined entry must still be EXCLUDED from the rendered digest"
true
) && ok
echo
if [ -s "$FAILFILE" ]; then
echo "wake digest-quarantine harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
exit 1
fi
echo "wake digest-quarantine harness: all invariants passed ($pass groups)"