fix(wake): #924 route dead-letter quarantine alarm via WAKE_ALARM_SINK_CMD with per-observed_seq dedup (G2a) (#929)
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
This commit was merged in pull request #929.
This commit is contained in:
@@ -36,6 +36,23 @@
|
||||
# longer wedge the whole drain (the live pilot defect: one malformed entry
|
||||
# exit-4'd the entire cumulative-state drain, delivering NOTHING).
|
||||
#
|
||||
# #924 (G2a fix — the #920 alarm was stderr/journal-LOCAL only): a dead-lettered
|
||||
# obligation is STORE-ACCOUNTED (§2.3), so the reconciler NEVER re-flags it — a
|
||||
# journal-local-only alarm means an unattended operator can PERMANENTLY MISS a
|
||||
# real obligation (the exact G2a silent-degradation shape the canon exists to
|
||||
# prevent). FIX: the per-entry quarantine alarm now ALSO routes through
|
||||
# WAKE_ALARM_SINK_CMD — the SAME pluggable off-host alarm-sink adapter beacon.sh
|
||||
# (W6/#910) uses: operator target resolved by-name inside the adapter, fail-
|
||||
# closed (unconfigured OR unreachable => a LOUD diagnostic, never a silent
|
||||
# no-alarm). The existing stderr diagnostic is KEPT (local + off-host, never
|
||||
# either/or). Per-observed_seq DEDUP (the entry's durable identity — entries
|
||||
# carry no per-entry wake_id; observed_seq is store.sh's sole-allocator
|
||||
# monotonic id, #908) via a durable alarmed-set file under STATE_DIR ensures a
|
||||
# still-dead-lettered entry is alarmed off-host EXACTLY ONCE, never once per
|
||||
# re-render (the digest re-renders the cumulative unacked set every drain
|
||||
# tick). A NEW distinct dead-lettered entry (a new observed_seq) still routes
|
||||
# its own one alarm. See _dlq_route_alarm / _dlq_already_alarmed / _dlq_mark_alarmed.
|
||||
#
|
||||
# #920 (amended FIX 2 — reconciler enumerations are ORIENTATION-tier): a
|
||||
# reconciler ENUMERATION carries locators.reconciled==true (set ONLY by
|
||||
# reconcile.sh). Enumerations are self-orienting STATE-POINTERS, never
|
||||
@@ -88,16 +105,28 @@ Options:
|
||||
Exit codes:
|
||||
0 digest rendered. Any render-refused ACTIONABLE entry (no §2.1 hard locator)
|
||||
is QUARANTINED — DEAD-LETTERED to $STATE_DIR/dead-letter.jsonl + a loud
|
||||
per-entry alarm — and EXCLUDED; the rest of the cumulative set still renders
|
||||
(#920: fail-loud is per-entry, never a whole-drain wedge). Reconciler
|
||||
enumerations (locators.reconciled==true) render ORIENTATION-tier, gate-exempt.
|
||||
per-entry alarm (stderr AND off-host via WAKE_ALARM_SINK_CMD, deduped by
|
||||
observed_seq, #924) — and EXCLUDED; the rest of the cumulative set still
|
||||
renders (#920: fail-loud is per-entry, never a whole-drain wedge). An
|
||||
unconfigured/unreachable WAKE_ALARM_SINK_CMD is itself a LOUD per-entry
|
||||
diagnostic (#924/G2a) — it never wedges the whole render either.
|
||||
Reconciler enumerations (locators.reconciled==true) render ORIENTATION-tier,
|
||||
gate-exempt.
|
||||
2 usage error.
|
||||
3 jq is required but missing.
|
||||
|
||||
Environment:
|
||||
WAKE_STATE_HOME override base state dir (XDG by default).
|
||||
WAKE_AGENT per-agent queue namespace / identity.
|
||||
WAKE_LANE default lane label.
|
||||
WAKE_STATE_HOME override base state dir (XDG by default).
|
||||
WAKE_AGENT per-agent queue namespace / identity.
|
||||
WAKE_LANE default lane label.
|
||||
WAKE_ALARM_SINK_CMD Pluggable off-host alarm route (SAME adapter beacon.sh/W6
|
||||
uses) for the #920 dead-letter QUARANTINE alarm (#924).
|
||||
Operator target resolved by-name inside the command.
|
||||
Fail-closed: unconfigured or a non-zero exit is a LOUD
|
||||
per-entry diagnostic (never a silent no-alarm host), but
|
||||
does not itself fail the overall render (per-entry, not
|
||||
whole-drain — mirrors the existing dead-letter-write
|
||||
fail-loud-but-non-wedging behavior).
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -313,14 +342,116 @@ _actionable_tier() {
|
||||
printf '0'
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #924 (G2a fix) — off-host dead-letter ALARM routing + per-observed_seq DEDUP.
|
||||
#
|
||||
# The #920 quarantine alarm was stderr/journal-LOCAL only. A dead-lettered entry
|
||||
# is STORE-ACCOUNTED (§2.3) so the reconciler never re-flags it: journal-local
|
||||
# visibility alone means an unattended operator can PERMANENTLY MISS a real
|
||||
# obligation. FIX: route the SAME per-entry alarm through WAKE_ALARM_SINK_CMD —
|
||||
# the pluggable off-host adapter beacon.sh (W6/#910) already defines (operator
|
||||
# target resolved by-name inside the adapter command, fail-closed) — IN ADDITION
|
||||
# to (never instead of) the existing stderr diagnostic.
|
||||
#
|
||||
# DEDUP: entries carry no per-entry wake_id (that field is the per-RENDER ack
|
||||
# copy-run id minted by sign.sh); the entry's durable identity is its
|
||||
# observed_seq (store.sh's sole monotonic allocator, #908 — stable for the life
|
||||
# of the entry). A durable alarmed-set file under STATE_DIR (one observed_seq
|
||||
# per line, atomic-written) records which observed_seqs have already been
|
||||
# routed off-host, so a still-dead-lettered entry re-drained every timer tick —
|
||||
# or across a process restart, since the marker is durable, not in-memory — is
|
||||
# alarmed off-host EXACTLY ONCE. A NEW distinct dead-lettered entry (a new
|
||||
# observed_seq) is not in the set, so it still routes its own one alarm.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# _dlq_alarmed_file — path to the durable per-entry off-host-alarm DEDUP marker
|
||||
# set. Lives alongside dead-letter.jsonl under the same per-agent STATE_DIR.
|
||||
_dlq_alarmed_file() { printf '%s/dead-letter-alarmed.set' "$STATE_DIR"; }
|
||||
|
||||
# _dlq_already_alarmed SEQ — true iff SEQ (observed_seq) already has a durable
|
||||
# alarmed marker, i.e. its off-host alarm has already been routed at least once.
|
||||
_dlq_already_alarmed() {
|
||||
local seq="$1" f
|
||||
f="$(_dlq_alarmed_file)"
|
||||
[ -s "$f" ] && grep -qxF "$seq" "$f" 2>/dev/null
|
||||
}
|
||||
|
||||
# _dlq_mark_alarmed SEQ — durably record that SEQ has been alarmed off-host.
|
||||
# Idempotent append + atomic write (same read-existing/append/atomic-write shape
|
||||
# as _quarantine_entry's own dead-letter-ledger append), so the marker survives
|
||||
# across drains AND restarts (it is durable STATE_DIR content, never an
|
||||
# in-process cache).
|
||||
_dlq_mark_alarmed() {
|
||||
local seq="$1" f existing
|
||||
f="$(_dlq_alarmed_file)"
|
||||
existing="$(cat "$f" 2>/dev/null || true)"
|
||||
printf '%s\n%s\n' "$existing" "$seq" | grep -v '^[[:space:]]*$' | _atomic_write "$f"
|
||||
}
|
||||
|
||||
# _dlq_route_alarm LINE SEQ — route ONE dead-letter alarm off-host via the
|
||||
# pluggable WAKE_ALARM_SINK_CMD adapter (`sh -c "$WAKE_ALARM_SINK_CMD"`, payload
|
||||
# JSON on stdin) — REUSING beacon.sh's exact adapter contract (§1.4: the
|
||||
# operator's command resolves its target BY NAME internally; this framework
|
||||
# file inlines no endpoint/secret). FAIL-CLOSED, mirroring beacon.sh's
|
||||
# _fire_alarm: an UNCONFIGURED (unset) or UNREACHABLE (non-zero exit) target is
|
||||
# a LOUD stderr diagnostic — never a silent no-alarm host. Returns non-zero on
|
||||
# either failure so the caller can skip marking the entry alarmed (an
|
||||
# unrouted alarm must be retried next drain, not falsely deduped away).
|
||||
#
|
||||
# NB: this failure is intentionally NOT propagated as a whole-render exit-code
|
||||
# failure (cmd_render still exits 0) — it is a PER-ENTRY fail-loud diagnostic,
|
||||
# exactly like a dead-letter ledger WRITE failure above: one bad/missing alarm
|
||||
# target must never wedge delivery of the rest of the cumulative-state drain
|
||||
# (#920's core fix). The loud diagnostic — not a process exit code — is the
|
||||
# fail-loud signal here.
|
||||
_dlq_route_alarm() {
|
||||
local line="$1" seq="$2" seq_json payload
|
||||
case "$seq" in
|
||||
'' | *[!0-9]*) seq_json='null' ;;
|
||||
*) seq_json="$seq" ;;
|
||||
esac
|
||||
payload="$(jq -cn --argjson seq "$seq_json" --argjson entry "$line" \
|
||||
'{kind:"wake-dead-letter-alarm", observed_seq:$seq, entry:$entry}' 2>/dev/null)"
|
||||
[ -n "$payload" ] || payload="$(printf '{"kind":"wake-dead-letter-alarm","observed_seq":%s}' "$seq_json")"
|
||||
|
||||
if [ -z "${WAKE_ALARM_SINK_CMD:-}" ]; then
|
||||
echo "digest.sh: FAIL LOUD (#924/G2a) — dead-letter entry at observed_seq=$seq is QUARANTINED (store-accounted; the reconciler will NEVER re-flag it) but WAKE_ALARM_SINK_CMD is UNSET: no off-host alarm target is configured. A journal-local-only diagnostic here is a silent no-alarm host — the operator can PERMANENTLY miss this obligation. Configure WAKE_ALARM_SINK_CMD (the same adapter beacon.sh/W6 uses)." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! printf '%s\n' "$payload" | sh -c "$WAKE_ALARM_SINK_CMD"; then
|
||||
echo "digest.sh: FAIL LOUD (#924/G2a) — dead-letter entry at observed_seq=$seq is QUARANTINED (store-accounted; the reconciler will NEVER re-flag it) but the off-host alarm sink is UNREACHABLE (WAKE_ALARM_SINK_CMD exited non-zero): the alarm did NOT reach a human/other-host. A journal-local-only diagnostic risks a PERMANENT silent miss of this obligation." >&2
|
||||
return 1
|
||||
fi
|
||||
echo "digest.sh: OFF-HOST ALARM ROUTED (#924/G2a) — dead-letter at observed_seq=$seq routed via WAKE_ALARM_SINK_CMD." >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
# _dlq_alarm_offhost LINE SEQ — the dedup-gated entry point _quarantine_entry
|
||||
# calls: route the off-host alarm for SEQ iff it has not already been alarmed
|
||||
# (durable dedup), marking it alarmed only on a SUCCESSFUL route (a failed
|
||||
# route is retried on the next drain, never falsely suppressed).
|
||||
_dlq_alarm_offhost() {
|
||||
local line="$1" seq="$2"
|
||||
if _dlq_already_alarmed "$seq"; then
|
||||
return 0
|
||||
fi
|
||||
if _dlq_route_alarm "$line" "$seq"; then
|
||||
_dlq_mark_alarmed "$seq"
|
||||
fi
|
||||
}
|
||||
|
||||
# _quarantine_entry LINE SEQ — QUARANTINE a render-refused entry (#920): append it
|
||||
# verbatim to the durable dead-letter ledger ($STATE_DIR/dead-letter.jsonl, atomic
|
||||
# write) and raise a LOUD per-entry fail-loud alarm on stderr. The entry is thereby
|
||||
# accounted-for (never silently dropped) without wedging the rest of the
|
||||
# cumulative-state drain. The append is idempotent (a still-pending malformed entry
|
||||
# is re-drained every timer tick) so the ledger stays bounded. A dead-letter write
|
||||
# failure is itself alarmed but never aborts the drain — the good entries MUST
|
||||
# still deliver (availability is the whole point of #920).
|
||||
# write) and raise a LOUD per-entry fail-loud alarm on stderr, PLUS route the SAME
|
||||
# alarm off-host via WAKE_ALARM_SINK_CMD, deduped by observed_seq (#924 — never
|
||||
# either/or; see the block above). The entry is thereby accounted-for (never
|
||||
# silently dropped) without wedging the rest of the cumulative-state drain. The
|
||||
# ledger append is idempotent (a still-pending malformed entry is re-drained every
|
||||
# timer tick) so the ledger stays bounded; the off-host alarm is independently
|
||||
# deduped by observed_seq so it fires ONCE regardless of ledger-append idempotency.
|
||||
# A dead-letter write failure is itself alarmed (both legs) but never aborts the
|
||||
# drain — the good entries MUST still deliver (availability is the whole point of
|
||||
# #920).
|
||||
_quarantine_entry() {
|
||||
local line="$1" seq="$2" dlq="$STATE_DIR/dead-letter.jsonl"
|
||||
if [ ! -f "$dlq" ] || ! grep -qxF "$line" "$dlq" 2>/dev/null; then
|
||||
@@ -328,10 +459,18 @@ _quarantine_entry() {
|
||||
existing="$(cat "$dlq" 2>/dev/null || true)"
|
||||
if ! printf '%s\n%s\n' "$existing" "$line" | grep -v '^[[:space:]]*$' | _atomic_write "$dlq"; then
|
||||
echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator AND the durable dead-letter write to $dlq FAILED. The entry is EXCLUDED from this digest and loudly surfaced here; investigate immediately." >&2
|
||||
# #924: the dlq ledger write failing must NOT suppress the off-host route
|
||||
# — that would silently fall back to a journal-local-only alarm, exactly
|
||||
# the G2a hazard this fix closes.
|
||||
_dlq_alarm_offhost "$line" "$seq"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator (repo+issue#/40-char SHA/file:anchor). DEAD-LETTERED to $dlq and EXCLUDED from this digest; the rest of the cumulative set still renders (no head-of-line block). Re-feed the source with a valid hard locator." >&2
|
||||
# #924 (G2a): route the SAME per-entry alarm off-host too, deduped by
|
||||
# observed_seq so a still-dead-lettered entry re-drained every tick is
|
||||
# alarmed off-host EXACTLY ONCE (never once per re-render).
|
||||
_dlq_alarm_offhost "$line" "$seq"
|
||||
}
|
||||
|
||||
cmd_render() {
|
||||
|
||||
Reference in New Issue
Block a user