From 347c1d57c1f19b5fd9cd18d342ebe9eee1fe2082 Mon Sep 17 00:00:00 2001 From: "jason.woltje" Date: Sun, 26 Jul 2026 11:45:09 +0000 Subject: [PATCH] fix(wake): #924 route dead-letter quarantine alarm via WAKE_ALARM_SINK_CMD with per-observed_seq dedup (G2a) (#929) Co-authored-by: jason.woltje Co-committed-by: jason.woltje --- .../mosaic/framework/tools/wake/digest.sh | 163 ++++++++++++++++-- .../mosaic/framework/tools/wake/manifest.txt | 34 +++- .../tools/wake/test-wake-digest-quarantine.sh | 136 +++++++++++++++ 3 files changed, 317 insertions(+), 16 deletions(-) diff --git a/packages/mosaic/framework/tools/wake/digest.sh b/packages/mosaic/framework/tools/wake/digest.sh index beab0d31..afd6d453 100755 --- a/packages/mosaic/framework/tools/wake/digest.sh +++ b/packages/mosaic/framework/tools/wake/digest.sh @@ -36,6 +36,23 @@ # longer wedge the whole drain (the live pilot defect: one malformed entry # exit-4'd the entire cumulative-state drain, delivering NOTHING). # +# #924 (G2a fix — the #920 alarm was stderr/journal-LOCAL only): a dead-lettered +# obligation is STORE-ACCOUNTED (§2.3), so the reconciler NEVER re-flags it — a +# journal-local-only alarm means an unattended operator can PERMANENTLY MISS a +# real obligation (the exact G2a silent-degradation shape the canon exists to +# prevent). FIX: the per-entry quarantine alarm now ALSO routes through +# WAKE_ALARM_SINK_CMD — the SAME pluggable off-host alarm-sink adapter beacon.sh +# (W6/#910) uses: operator target resolved by-name inside the adapter, fail- +# closed (unconfigured OR unreachable => a LOUD diagnostic, never a silent +# no-alarm). The existing stderr diagnostic is KEPT (local + off-host, never +# either/or). Per-observed_seq DEDUP (the entry's durable identity — entries +# carry no per-entry wake_id; observed_seq is store.sh's sole-allocator +# monotonic id, #908) via a durable alarmed-set file under STATE_DIR ensures a +# still-dead-lettered entry is alarmed off-host EXACTLY ONCE, never once per +# re-render (the digest re-renders the cumulative unacked set every drain +# tick). A NEW distinct dead-lettered entry (a new observed_seq) still routes +# its own one alarm. See _dlq_route_alarm / _dlq_already_alarmed / _dlq_mark_alarmed. +# # #920 (amended FIX 2 — reconciler enumerations are ORIENTATION-tier): a # reconciler ENUMERATION carries locators.reconciled==true (set ONLY by # reconcile.sh). Enumerations are self-orienting STATE-POINTERS, never @@ -88,16 +105,28 @@ Options: Exit codes: 0 digest rendered. Any render-refused ACTIONABLE entry (no §2.1 hard locator) is QUARANTINED — DEAD-LETTERED to $STATE_DIR/dead-letter.jsonl + a loud - per-entry alarm — and EXCLUDED; the rest of the cumulative set still renders - (#920: fail-loud is per-entry, never a whole-drain wedge). Reconciler - enumerations (locators.reconciled==true) render ORIENTATION-tier, gate-exempt. + per-entry alarm (stderr AND off-host via WAKE_ALARM_SINK_CMD, deduped by + observed_seq, #924) — and EXCLUDED; the rest of the cumulative set still + renders (#920: fail-loud is per-entry, never a whole-drain wedge). An + unconfigured/unreachable WAKE_ALARM_SINK_CMD is itself a LOUD per-entry + diagnostic (#924/G2a) — it never wedges the whole render either. + Reconciler enumerations (locators.reconciled==true) render ORIENTATION-tier, + gate-exempt. 2 usage error. 3 jq is required but missing. Environment: - WAKE_STATE_HOME override base state dir (XDG by default). - WAKE_AGENT per-agent queue namespace / identity. - WAKE_LANE default lane label. + WAKE_STATE_HOME override base state dir (XDG by default). + WAKE_AGENT per-agent queue namespace / identity. + WAKE_LANE default lane label. + WAKE_ALARM_SINK_CMD Pluggable off-host alarm route (SAME adapter beacon.sh/W6 + uses) for the #920 dead-letter QUARANTINE alarm (#924). + Operator target resolved by-name inside the command. + Fail-closed: unconfigured or a non-zero exit is a LOUD + per-entry diagnostic (never a silent no-alarm host), but + does not itself fail the overall render (per-entry, not + whole-drain — mirrors the existing dead-letter-write + fail-loud-but-non-wedging behavior). EOF } @@ -313,14 +342,116 @@ _actionable_tier() { printf '0' } +# --------------------------------------------------------------------------- +# #924 (G2a fix) — off-host dead-letter ALARM routing + per-observed_seq DEDUP. +# +# The #920 quarantine alarm was stderr/journal-LOCAL only. A dead-lettered entry +# is STORE-ACCOUNTED (§2.3) so the reconciler never re-flags it: journal-local +# visibility alone means an unattended operator can PERMANENTLY MISS a real +# obligation. FIX: route the SAME per-entry alarm through WAKE_ALARM_SINK_CMD — +# the pluggable off-host adapter beacon.sh (W6/#910) already defines (operator +# target resolved by-name inside the adapter command, fail-closed) — IN ADDITION +# to (never instead of) the existing stderr diagnostic. +# +# DEDUP: entries carry no per-entry wake_id (that field is the per-RENDER ack +# copy-run id minted by sign.sh); the entry's durable identity is its +# observed_seq (store.sh's sole monotonic allocator, #908 — stable for the life +# of the entry). A durable alarmed-set file under STATE_DIR (one observed_seq +# per line, atomic-written) records which observed_seqs have already been +# routed off-host, so a still-dead-lettered entry re-drained every timer tick — +# or across a process restart, since the marker is durable, not in-memory — is +# alarmed off-host EXACTLY ONCE. A NEW distinct dead-lettered entry (a new +# observed_seq) is not in the set, so it still routes its own one alarm. +# --------------------------------------------------------------------------- + +# _dlq_alarmed_file — path to the durable per-entry off-host-alarm DEDUP marker +# set. Lives alongside dead-letter.jsonl under the same per-agent STATE_DIR. +_dlq_alarmed_file() { printf '%s/dead-letter-alarmed.set' "$STATE_DIR"; } + +# _dlq_already_alarmed SEQ — true iff SEQ (observed_seq) already has a durable +# alarmed marker, i.e. its off-host alarm has already been routed at least once. +_dlq_already_alarmed() { + local seq="$1" f + f="$(_dlq_alarmed_file)" + [ -s "$f" ] && grep -qxF "$seq" "$f" 2>/dev/null +} + +# _dlq_mark_alarmed SEQ — durably record that SEQ has been alarmed off-host. +# Idempotent append + atomic write (same read-existing/append/atomic-write shape +# as _quarantine_entry's own dead-letter-ledger append), so the marker survives +# across drains AND restarts (it is durable STATE_DIR content, never an +# in-process cache). +_dlq_mark_alarmed() { + local seq="$1" f existing + f="$(_dlq_alarmed_file)" + existing="$(cat "$f" 2>/dev/null || true)" + printf '%s\n%s\n' "$existing" "$seq" | grep -v '^[[:space:]]*$' | _atomic_write "$f" +} + +# _dlq_route_alarm LINE SEQ — route ONE dead-letter alarm off-host via the +# pluggable WAKE_ALARM_SINK_CMD adapter (`sh -c "$WAKE_ALARM_SINK_CMD"`, payload +# JSON on stdin) — REUSING beacon.sh's exact adapter contract (§1.4: the +# operator's command resolves its target BY NAME internally; this framework +# file inlines no endpoint/secret). FAIL-CLOSED, mirroring beacon.sh's +# _fire_alarm: an UNCONFIGURED (unset) or UNREACHABLE (non-zero exit) target is +# a LOUD stderr diagnostic — never a silent no-alarm host. Returns non-zero on +# either failure so the caller can skip marking the entry alarmed (an +# unrouted alarm must be retried next drain, not falsely deduped away). +# +# NB: this failure is intentionally NOT propagated as a whole-render exit-code +# failure (cmd_render still exits 0) — it is a PER-ENTRY fail-loud diagnostic, +# exactly like a dead-letter ledger WRITE failure above: one bad/missing alarm +# target must never wedge delivery of the rest of the cumulative-state drain +# (#920's core fix). The loud diagnostic — not a process exit code — is the +# fail-loud signal here. +_dlq_route_alarm() { + local line="$1" seq="$2" seq_json payload + case "$seq" in + '' | *[!0-9]*) seq_json='null' ;; + *) seq_json="$seq" ;; + esac + payload="$(jq -cn --argjson seq "$seq_json" --argjson entry "$line" \ + '{kind:"wake-dead-letter-alarm", observed_seq:$seq, entry:$entry}' 2>/dev/null)" + [ -n "$payload" ] || payload="$(printf '{"kind":"wake-dead-letter-alarm","observed_seq":%s}' "$seq_json")" + + if [ -z "${WAKE_ALARM_SINK_CMD:-}" ]; then + echo "digest.sh: FAIL LOUD (#924/G2a) — dead-letter entry at observed_seq=$seq is QUARANTINED (store-accounted; the reconciler will NEVER re-flag it) but WAKE_ALARM_SINK_CMD is UNSET: no off-host alarm target is configured. A journal-local-only diagnostic here is a silent no-alarm host — the operator can PERMANENTLY miss this obligation. Configure WAKE_ALARM_SINK_CMD (the same adapter beacon.sh/W6 uses)." >&2 + return 1 + fi + if ! printf '%s\n' "$payload" | sh -c "$WAKE_ALARM_SINK_CMD"; then + echo "digest.sh: FAIL LOUD (#924/G2a) — dead-letter entry at observed_seq=$seq is QUARANTINED (store-accounted; the reconciler will NEVER re-flag it) but the off-host alarm sink is UNREACHABLE (WAKE_ALARM_SINK_CMD exited non-zero): the alarm did NOT reach a human/other-host. A journal-local-only diagnostic risks a PERMANENT silent miss of this obligation." >&2 + return 1 + fi + echo "digest.sh: OFF-HOST ALARM ROUTED (#924/G2a) — dead-letter at observed_seq=$seq routed via WAKE_ALARM_SINK_CMD." >&2 + return 0 +} + +# _dlq_alarm_offhost LINE SEQ — the dedup-gated entry point _quarantine_entry +# calls: route the off-host alarm for SEQ iff it has not already been alarmed +# (durable dedup), marking it alarmed only on a SUCCESSFUL route (a failed +# route is retried on the next drain, never falsely suppressed). +_dlq_alarm_offhost() { + local line="$1" seq="$2" + if _dlq_already_alarmed "$seq"; then + return 0 + fi + if _dlq_route_alarm "$line" "$seq"; then + _dlq_mark_alarmed "$seq" + fi +} + # _quarantine_entry LINE SEQ — QUARANTINE a render-refused entry (#920): append it # verbatim to the durable dead-letter ledger ($STATE_DIR/dead-letter.jsonl, atomic -# write) and raise a LOUD per-entry fail-loud alarm on stderr. The entry is thereby -# accounted-for (never silently dropped) without wedging the rest of the -# cumulative-state drain. The append is idempotent (a still-pending malformed entry -# is re-drained every timer tick) so the ledger stays bounded. A dead-letter write -# failure is itself alarmed but never aborts the drain — the good entries MUST -# still deliver (availability is the whole point of #920). +# write) and raise a LOUD per-entry fail-loud alarm on stderr, PLUS route the SAME +# alarm off-host via WAKE_ALARM_SINK_CMD, deduped by observed_seq (#924 — never +# either/or; see the block above). The entry is thereby accounted-for (never +# silently dropped) without wedging the rest of the cumulative-state drain. The +# ledger append is idempotent (a still-pending malformed entry is re-drained every +# timer tick) so the ledger stays bounded; the off-host alarm is independently +# deduped by observed_seq so it fires ONCE regardless of ledger-append idempotency. +# A dead-letter write failure is itself alarmed (both legs) but never aborts the +# drain — the good entries MUST still deliver (availability is the whole point of +# #920). _quarantine_entry() { local line="$1" seq="$2" dlq="$STATE_DIR/dead-letter.jsonl" if [ ! -f "$dlq" ] || ! grep -qxF "$line" "$dlq" 2>/dev/null; then @@ -328,10 +459,18 @@ _quarantine_entry() { existing="$(cat "$dlq" 2>/dev/null || true)" if ! printf '%s\n%s\n' "$existing" "$line" | grep -v '^[[:space:]]*$' | _atomic_write "$dlq"; then echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator AND the durable dead-letter write to $dlq FAILED. The entry is EXCLUDED from this digest and loudly surfaced here; investigate immediately." >&2 + # #924: the dlq ledger write failing must NOT suppress the off-host route + # — that would silently fall back to a journal-local-only alarm, exactly + # the G2a hazard this fix closes. + _dlq_alarm_offhost "$line" "$seq" return 0 fi fi echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator (repo+issue#/40-char SHA/file:anchor). DEAD-LETTERED to $dlq and EXCLUDED from this digest; the rest of the cumulative set still renders (no head-of-line block). Re-feed the source with a valid hard locator." >&2 + # #924 (G2a): route the SAME per-entry alarm off-host too, deduped by + # observed_seq so a still-dead-lettered entry re-drained every tick is + # alarmed off-host EXACTLY ONCE (never once per re-render). + _dlq_alarm_offhost "$line" "$seq" } cmd_render() { diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index ef4b2638..58f18388 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -98,8 +98,32 @@ # seen-ledger nor observed_seq, so the source is re-enumerated next # cycle — no obligation loss). Files changed: _wake-common.sh, # store.sh, detector.sh (+ tests). +# 0.6.6 #924 digest.sh dead-letter QUARANTINE alarm — G2a fix (dragon-lin +# cure-verification follow-up on #920/PR #922). The #920 per-entry +# quarantine alarm was stderr/journal-LOCAL only; a dead-lettered +# entry is STORE-ACCOUNTED (§2.3) so the reconciler never re-flags +# it, so journal-local-only visibility meant an unattended operator +# could PERMANENTLY MISS a real obligation (G2a silent-degradation). +# FIX: the SAME per-entry quarantine alarm now ALSO routes through +# WAKE_ALARM_SINK_CMD — REUSING beacon.sh's (W6/#910) exact +# pluggable off-host alarm-sink adapter contract (operator target +# resolved by-name inside the adapter, fail-closed) — IN ADDITION +# to (never instead of) the existing stderr diagnostic. Per- +# observed_seq DEDUP (entries carry no per-entry wake_id; the +# entry's durable identity is its store-allocated observed_seq, +# #908) via a durable alarmed-set file under STATE_DIR +# (dead-letter-alarmed.set, atomic-written) ensures a still-dead- +# lettered entry is alarmed off-host EXACTLY ONCE per drain/restart, +# never once per re-render; a NEW distinct dead-lettered entry +# still routes its own one alarm. An unconfigured/unreachable +# WAKE_ALARM_SINK_CMD is a LOUD per-entry stderr diagnostic +# (mirrors beacon.sh's fail-closed wording) but does NOT itself +# fail the whole render (per-entry fail-loud, never a whole-drain +# wedge — #920's core property is preserved). digest.sh is the +# ONLY file changed; store.sh/beacon.sh/reconcile.sh are +# UNCHANGED (beacon.sh's adapter contract is reused, not modified). component=wake -version=0.6.5 +version=0.6.6 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version @@ -117,9 +141,11 @@ schema_max=1 # digest.sh A3 — cumulative-state digest renderer (hard locators, # two-tier trust, injection/secret scrub). PER-ENTRY # quarantine: a render-refused entry is dead-lettered + -# alarmed + excluded, the rest still renders (no head-of-line -# block); reconciler enumerations (reconciled==true) render -# ORIENTATION-tier, gate-exempt. (W3, #920) +# alarmed (stderr AND off-host via WAKE_ALARM_SINK_CMD, +# deduped by observed_seq, #924) + excluded, the rest still +# renders (no head-of-line block); reconciler enumerations +# (reconciled==true) render ORIENTATION-tier, gate-exempt. +# (W3, #920, #924) # sign.sh A5 — non-circular HMAC signer (independent wake_id, # load_credentials by-name; fills the hmac placeholder). (W3) # detector.sh A1 — per-host single-instance delta-gated detector daemon diff --git a/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh b/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh index 61db92d0..002d5b4f 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh @@ -26,6 +26,33 @@ # `claim` (a consequential fact) with no hard locator is STILL gated # (quarantined), and the reconciled exemption does not leak to it. (§2.1) # +# #924 (G2a fix — the #920 alarm was stderr/journal-LOCAL only, a permanent +# silent-miss hazard since a dead-lettered entry is store-accounted and the +# reconciler never re-flags it): the SAME per-entry alarm now ALSO routes +# off-host via WAKE_ALARM_SINK_CMD (beacon.sh's W6/#910 pluggable adapter, +# REUSED verbatim), deduped by observed_seq (the entry's durable identity). +# Q6 (a) ONE off-host alarm + stderr diagnostic STILL fires: a dead-lettered +# entry routes EXACTLY ONE alarm to a captured WAKE_ALARM_SINK_CMD +# (payload names observed_seq), AND the #920 stderr diagnostic still +# fires (local + off-host, never either/or). +# Q7 (b) RE-DRAIN DEDUP: re-draining the SAME still-dead-lettered entry N +# times routes ZERO additional off-host alarms (durable dedup by +# observed_seq survives across separate digest.sh invocations, i.e. +# across drains/restarts, since each invocation is a fresh process). +# Q8 (c) NEW ENTRY OWN ALARM: a NEW distinct dead-lettered entry (a new +# observed_seq) routes its OWN one alarm — dedup is per-entry, not a +# global "alarm already fired at all" latch. +# Q9 (d) FAIL-CLOSED: WAKE_ALARM_SINK_CMD unconfigured OR unreachable (exit +# non-zero) is a LOUD stderr diagnostic (mirrors beacon.sh's +# fail-closed wording) — never a silent no-alarm host. Per-entry, not +# whole-drain: render still exits 0 (#920's no-head-of-line-block +# property is preserved even when the off-host leg itself fails). +# +# RED-FIRST: Q6-Q9 all go RED against the pre-#924 (stderr-only) digest.sh — +# it never references WAKE_ALARM_SINK_CMD at all, so no payload is EVER routed +# (Q6/Q7/Q8 all see 0 captured alarms) and no "FAIL LOUD ... alarm sink" +# diagnostic exists to fire (Q9). +# # Hermetic: feeds controlled JSONL via `digest.sh render --from-file` — NO store, # NO network, NO openssl (so it runs identically under the CI openssl-mask). # @@ -57,6 +84,17 @@ ok() { pass=$((pass + 1)); } # A 40-hex sha = a valid §2.1 hard locator. SHA40="abcdef0123456789abcdef0123456789abcdef01" +# capture-alarm (#924): a REACHABLE off-host alarm route that APPENDS the +# routed payload (one JSON line per invocation) to $ALARM_OUT, so a test can +# count exactly how many off-host alarms fired across one or more digest.sh +# invocations. Mirrors test-wake-beacon.sh's capture-alarm idiom. +CAPTURE_ALARM="$TMP_ROOT/capture-alarm.sh" +cat >"$CAPTURE_ALARM" <<'EOF' +#!/usr/bin/env bash +cat >>"$ALARM_OUT" +EOF +chmod +x "$CAPTURE_ALARM" + # fresh_home NAME — a fresh WAKE_STATE_HOME dir, echoed. fresh_home() { local d="$TMP_ROOT/$1" @@ -165,6 +203,104 @@ echo "== Q5: claim-precedence gated — a non-actionable-class entry carrying a true ) && ok +echo "== Q6 (a): dead-lettered entry routes EXACTLY ONE off-host alarm (payload names observed_seq) + stderr diagnostic STILL fires ==" +( + home="$(fresh_home q6)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + f="$TMP_ROOT/q6.jsonl" + printf '%s\n' '{"observed_seq":21,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q6","path":"x.md","observed_hash":"aaaa"},"emit_ts":1}' >"$f" + ALARM_OUT="$TMP_ROOT/q6.alarm.jsonl" + export ALARM_OUT + : >"$ALARM_OUT" + export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" + err="$TMP_ROOT/q6.err" + out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "Q6: render must still EXIT 0 (per-entry quarantine, not whole-drain wedge), got rc=$rc" + n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)" + [ "$n" = "1" ] || fail_msg "Q6: EXACTLY ONE off-host alarm must route for the dead-lettered entry (got $n) [$(cat "$ALARM_OUT" 2>/dev/null)]" + grep -q '"observed_seq":21' "$ALARM_OUT" 2>/dev/null || fail_msg "Q6: the routed alarm payload must name the entry's observed_seq (21)" + grep -qi 'QUARANTINE' "$err" || fail_msg "Q6: the existing #920 stderr diagnostic must STILL fire (local + off-host, not either/or)" + printf '%s' "$out" | grep -q 'DLQ-Q6' && fail_msg "Q6: the quarantined entry must still be EXCLUDED from the rendered digest" + true +) && ok + +echo "== Q7 (b): re-draining the SAME still-dead-lettered entry N times routes ZERO additional off-host alarms (dedup by observed_seq) ==" +( + home="$(fresh_home q7)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + f="$TMP_ROOT/q7.jsonl" + printf '%s\n' '{"observed_seq":22,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q7","path":"y.md","observed_hash":"bbbb"},"emit_ts":1}' >"$f" + ALARM_OUT="$TMP_ROOT/q7.alarm.jsonl" + export ALARM_OUT + : >"$ALARM_OUT" + export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" + for i in 1 2 3 4; do + "$DIGEST" render --from-file "$f" --agent default >/dev/null 2>"$TMP_ROOT/q7.err.$i" + done + n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)" + [ "$n" = "1" ] || fail_msg "Q7: re-draining the SAME dead-lettered entry 4x must route ONLY ONE off-host alarm total (durable dedup by observed_seq); got $n [$(cat "$ALARM_OUT" 2>/dev/null)]" + grep -qi 'QUARANTINE' "$TMP_ROOT/q7.err.4" || fail_msg "Q7: the #920 stderr diagnostic must STILL fire on every re-drain (only the off-host route is deduped)" +) && ok + +echo "== Q8 (c): a NEW distinct dead-lettered entry routes its OWN one alarm (dedup is per-entry, not a global latch) ==" +( + home="$(fresh_home q8)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + f1="$TMP_ROOT/q8a.jsonl" + f2="$TMP_ROOT/q8b.jsonl" + printf '%s\n' '{"observed_seq":31,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q8A","path":"a.md","observed_hash":"c1"},"emit_ts":1}' >"$f1" + printf '%s\n' '{"observed_seq":32,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q8B","path":"b.md","observed_hash":"c2"},"emit_ts":1}' >"$f2" + ALARM_OUT="$TMP_ROOT/q8.alarm.jsonl" + export ALARM_OUT + : >"$ALARM_OUT" + export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM" + "$DIGEST" render --from-file "$f1" --agent default >/dev/null 2>/dev/null + "$DIGEST" render --from-file "$f1" --agent default >/dev/null 2>/dev/null # re-drain seq 31 -> must NOT re-alarm + "$DIGEST" render --from-file "$f2" --agent default >/dev/null 2>/dev/null # NEW distinct seq 32 -> its own alarm + n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)" + [ "$n" = "2" ] || fail_msg "Q8: two DISTINCT dead-lettered entries must together route exactly 2 off-host alarms total (got $n) [$(cat "$ALARM_OUT" 2>/dev/null)]" + grep -q '"observed_seq":31' "$ALARM_OUT" 2>/dev/null || fail_msg "Q8: seq 31's alarm must be present" + grep -q '"observed_seq":32' "$ALARM_OUT" 2>/dev/null || fail_msg "Q8: seq 32's (the new distinct entry's) OWN alarm must be present" +) && ok + +echo "== Q9 (d): WAKE_ALARM_SINK_CMD unconfigured OR unreachable -> FAIL LOUD (never silent no-alarm); per-entry, render still exits 0 ==" +( + home="$(fresh_home q9a)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + f="$TMP_ROOT/q9.jsonl" + printf '%s\n' '{"observed_seq":41,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q9","path":"z.md","observed_hash":"dddd"},"emit_ts":1}' >"$f" + # (a) UNCONFIGURED alarm sink. + unset WAKE_ALARM_SINK_CMD + err_a="$TMP_ROOT/q9a.err" + out_a="$("$DIGEST" render --from-file "$f" --agent default 2>"$err_a")" + rc_a=$? + [ "$rc_a" -eq 0 ] || fail_msg "Q9a: per-entry quarantine must still exit 0 even when the off-host alarm sink is unconfigured (no whole-drain wedge), got rc=$rc_a" + grep -qi 'FAIL LOUD' "$err_a" || fail_msg "Q9a: an unconfigured off-host alarm target must FAIL LOUD on stderr [$(cat "$err_a")]" + grep -Eqi 'silent no-alarm|silent-miss|PERMANENTLY miss|permanent silent miss' "$err_a" || fail_msg "Q9a: the diagnostic must name the silent-miss hazard (G2a), mirroring beacon.sh's fail-closed wording [$(cat "$err_a")]" + printf '%s' "$out_a" | grep -q 'DLQ-Q9' && fail_msg "Q9a: the quarantined entry must still be EXCLUDED from the rendered digest" + true +) && ok +( + home2="$(fresh_home q9b)" + export WAKE_STATE_HOME="$home2" + unset WAKE_AGENT + f="$TMP_ROOT/q9.jsonl" + export WAKE_ALARM_SINK_CMD="false" + err_b="$TMP_ROOT/q9b.err" + out_b="$("$DIGEST" render --from-file "$f" --agent default 2>"$err_b")" + rc_b=$? + [ "$rc_b" -eq 0 ] || fail_msg "Q9b: per-entry quarantine must still exit 0 even when the off-host alarm sink is unreachable, got rc=$rc_b" + grep -qi 'FAIL LOUD' "$err_b" || fail_msg "Q9b: an unreachable off-host alarm target must FAIL LOUD on stderr [$(cat "$err_b")]" + grep -qi 'UNREACHABLE' "$err_b" || fail_msg "Q9b: the diagnostic must name the unreachable target [$(cat "$err_b")]" + printf '%s' "$out_b" | grep -q 'DLQ-Q9' && fail_msg "Q9b: the quarantined entry must still be EXCLUDED from the rendered digest" + true +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake digest-quarantine harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2